aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--INSTALL.md8
-rw-r--r--configure.in6
-rw-r--r--erts/aclocal.m4434
-rw-r--r--erts/doc/src/erl_nif.xml7
-rw-r--r--erts/doc/src/erlang.xml9
-rw-r--r--erts/doc/src/erts_alloc.xml66
-rw-r--r--erts/emulator/Makefile.in2
-rw-r--r--erts/emulator/beam/atom.c8
-rw-r--r--erts/emulator/beam/beam_bp.c12
-rw-r--r--erts/emulator/beam/beam_bp.h6
-rw-r--r--erts/emulator/beam/bif.c18
-rw-r--r--erts/emulator/beam/break.c7
-rw-r--r--erts/emulator/beam/dist.c16
-rw-r--r--erts/emulator/beam/dist.h4
-rw-r--r--erts/emulator/beam/erl_afit_alloc.c15
-rw-r--r--erts/emulator/beam/erl_alloc.c184
-rw-r--r--erts/emulator/beam/erl_alloc.h8
-rw-r--r--erts/emulator/beam/erl_alloc.types15
-rw-r--r--erts/emulator/beam/erl_alloc_util.c698
-rw-r--r--erts/emulator/beam/erl_alloc_util.h43
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.c972
-rw-r--r--erts/emulator/beam/erl_ao_firstfit_alloc.h60
-rw-r--r--erts/emulator/beam/erl_bestfit_alloc.c168
-rw-r--r--erts/emulator/beam/erl_bestfit_alloc.h3
-rw-r--r--erts/emulator/beam/erl_bif_ddll.c8
-rw-r--r--erts/emulator/beam/erl_bif_info.c92
-rw-r--r--erts/emulator/beam/erl_bits.c10
-rw-r--r--erts/emulator/beam/erl_cpu_topology.c4
-rw-r--r--erts/emulator/beam/erl_db.c103
-rw-r--r--erts/emulator/beam/erl_db.h8
-rw-r--r--erts/emulator/beam/erl_db_hash.c101
-rw-r--r--erts/emulator/beam/erl_db_tree.c24
-rw-r--r--erts/emulator/beam/erl_db_util.c13
-rw-r--r--erts/emulator/beam/erl_goodfit_alloc.c22
-rw-r--r--erts/emulator/beam/erl_init.c16
-rw-r--r--erts/emulator/beam/erl_instrument.c6
-rw-r--r--erts/emulator/beam/erl_lock_check.c1
-rw-r--r--erts/emulator/beam/erl_monitors.c12
-rw-r--r--erts/emulator/beam/erl_nif.c33
-rw-r--r--erts/emulator/beam/erl_nif.h3
-rw-r--r--erts/emulator/beam/erl_nif_api_funcs.h2
-rw-r--r--erts/emulator/beam/erl_node_tables.c6
-rw-r--r--erts/emulator/beam/erl_port_task.c49
-rw-r--r--erts/emulator/beam/erl_port_task.h8
-rw-r--r--erts/emulator/beam/erl_process.c257
-rw-r--r--erts/emulator/beam/erl_process.h4
-rw-r--r--erts/emulator/beam/erl_process_dump.c2
-rw-r--r--erts/emulator/beam/erl_process_lock.c15
-rw-r--r--erts/emulator/beam/erl_process_lock.h22
-rw-r--r--erts/emulator/beam/erl_smp.h973
-rw-r--r--erts/emulator/beam/erl_threads.h927
-rw-r--r--erts/emulator/beam/erl_time.h4
-rw-r--r--erts/emulator/beam/global.h25
-rw-r--r--erts/emulator/beam/io.c42
-rw-r--r--erts/emulator/beam/safe_hash.c10
-rw-r--r--erts/emulator/beam/sys.h47
-rw-r--r--erts/emulator/beam/time.c4
-rw-r--r--erts/emulator/beam/utils.c24
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c4
-rw-r--r--erts/emulator/sys/common/erl_check_io.c28
-rw-r--r--erts/emulator/sys/common/erl_poll.c94
-rw-r--r--erts/emulator/sys/unix/sys.c38
-rw-r--r--erts/emulator/sys/win32/erl_poll.c62
-rw-r--r--erts/emulator/sys/win32/sys.c49
-rw-r--r--erts/emulator/sys/win32/sys_interrupt.c4
-rw-r--r--erts/emulator/test/alloc_SUITE_data/allocator_test.h20
-rw-r--r--erts/emulator/test/alloc_SUITE_data/coalesce.c2
-rw-r--r--erts/emulator/test/alloc_SUITE_data/rbtree.c86
-rw-r--r--erts/emulator/test/binary_SUITE.erl41
-rw-r--r--erts/emulator/test/fun_SUITE.erl10
-rw-r--r--erts/emulator/test/match_spec_SUITE.erl61
-rw-r--r--erts/emulator/test/nif_SUITE.erl57
-rw-r--r--erts/emulator/test/nif_SUITE_data/nif_SUITE.c55
-rw-r--r--erts/emulator/test/send_term_SUITE.erl4
-rw-r--r--erts/epmd/src/epmd_int.h9
-rw-r--r--erts/epmd/src/epmd_srv.c20
-rw-r--r--erts/etc/common/erlexec.c3
-rw-r--r--erts/include/internal/ethr_atomics.h8932
-rw-r--r--erts/include/internal/ethr_internal.h6
-rw-r--r--erts/include/internal/ethr_mutex.h30
-rw-r--r--erts/include/internal/ethr_optimized_fallbacks.h175
-rw-r--r--erts/include/internal/ethread.h162
-rw-r--r--erts/include/internal/ethread_header_config.h.in118
-rw-r--r--erts/include/internal/gcc/ethr_atomic.h251
-rw-r--r--erts/include/internal/gcc/ethr_dw_atomic.h115
-rw-r--r--erts/include/internal/gcc/ethr_membar.h73
-rw-r--r--erts/include/internal/gcc/ethread.h20
-rw-r--r--erts/include/internal/i386/atomic.h334
-rw-r--r--erts/include/internal/i386/ethr_dw_atomic.h278
-rw-r--r--erts/include/internal/i386/ethr_membar.h114
-rw-r--r--erts/include/internal/i386/ethread.h8
-rw-r--r--erts/include/internal/i386/rwlock.h5
-rw-r--r--erts/include/internal/i386/spinlock.h27
-rw-r--r--erts/include/internal/libatomic_ops/ethr_atomic.h298
-rw-r--r--erts/include/internal/libatomic_ops/ethr_membar.h75
-rw-r--r--erts/include/internal/libatomic_ops/ethread.h14
-rw-r--r--erts/include/internal/ppc32/atomic.h148
-rw-r--r--erts/include/internal/ppc32/ethr_membar.h63
-rw-r--r--erts/include/internal/ppc32/ethread.h5
-rw-r--r--erts/include/internal/ppc32/rwlock.h15
-rw-r--r--erts/include/internal/ppc32/spinlock.h10
-rw-r--r--erts/include/internal/pthread/ethr_event.h8
-rw-r--r--erts/include/internal/sparc32/atomic.h194
-rw-r--r--erts/include/internal/sparc32/ethr_membar.h115
-rw-r--r--erts/include/internal/sparc32/ethread.h7
-rw-r--r--erts/include/internal/sparc32/rwlock.h17
-rw-r--r--erts/include/internal/sparc32/spinlock.h11
-rw-r--r--erts/include/internal/tile/atomic.h136
-rw-r--r--erts/include/internal/tile/ethr_membar.h35
-rw-r--r--erts/include/internal/tile/ethread.h5
-rw-r--r--erts/include/internal/win/ethr_atomic.h595
-rw-r--r--erts/include/internal/win/ethr_dw_atomic.h154
-rw-r--r--erts/include/internal/win/ethr_event.h16
-rw-r--r--erts/include/internal/win/ethr_membar.h145
-rw-r--r--erts/include/internal/win/ethread.h4
-rw-r--r--erts/lib_src/Makefile.in14
-rw-r--r--erts/lib_src/common/ethr_atomics.c4259
-rw-r--r--erts/lib_src/common/ethr_aux.c83
-rw-r--r--erts/lib_src/common/ethr_mutex.c5
-rw-r--r--erts/lib_src/pthread/ethr_x86_sse2_asm.c31
-rw-r--r--erts/lib_src/pthread/ethread.c102
-rwxr-xr-xerts/lib_src/utils/make_atomics_api2186
-rw-r--r--erts/lib_src/win/ethr_event.c22
-rw-r--r--erts/lib_src/win/ethread.c22
-rw-r--r--erts/preloaded/ebin/erlang.beambin27016 -> 26772 bytes
-rw-r--r--erts/preloaded/src/erlang.erl8
-rw-r--r--erts/test/ethread_SUITE.erl32
-rw-r--r--erts/test/ethread_SUITE_data/ethread_tests.c428
-rw-r--r--lib/asn1/c_src/Makefile31
-rw-r--r--lib/asn1/c_src/asn1_erl_driver.c1677
-rw-r--r--lib/asn1/c_src/asn1_erl_nif.c1079
-rw-r--r--lib/asn1/doc/src/asn1_spec.xmlsrc8
-rw-r--r--lib/asn1/doc/src/asn1_ug.xml50
-rw-r--r--lib/asn1/doc/src/asn1ct.xml29
-rw-r--r--lib/asn1/doc/src/asn1rt.xml65
-rw-r--r--lib/asn1/src/Makefile4
-rw-r--r--lib/asn1/src/asn1.app.src5
-rw-r--r--lib/asn1/src/asn1_app.erl2
-rw-r--r--lib/asn1/src/asn1_server.erl107
-rw-r--r--lib/asn1/src/asn1ct.erl12
-rw-r--r--lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl8
-rw-r--r--lib/asn1/src/asn1ct_constructed_per.erl116
-rw-r--r--lib/asn1/src/asn1ct_gen.erl30
-rw-r--r--lib/asn1/src/asn1ct_gen_ber_bin_v2.erl8
-rw-r--r--lib/asn1/src/asn1ct_gen_per.erl5
-rw-r--r--lib/asn1/src/asn1ct_gen_per_rt2ct.erl5
-rw-r--r--lib/asn1/src/asn1rt.erl38
-rw-r--r--lib/asn1/src/asn1rt_ber_bin_v2.erl104
-rw-r--r--lib/asn1/src/asn1rt_driver_handler.erl144
-rw-r--r--lib/asn1/src/asn1rt_nif.erl83
-rw-r--r--lib/asn1/src/asn1rt_per_bin.erl4
-rw-r--r--lib/asn1/src/asn1rt_per_bin_rt2ct.erl131
-rw-r--r--lib/asn1/test/asn1.cover1
-rw-r--r--lib/asn1/test/asn1_SUITE.erl.src6
-rw-r--r--lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src172
-rw-r--r--lib/asn1/test/ber_decode_error.erl6
-rw-r--r--lib/asn1/test/testPrim.erl21
-rw-r--r--lib/compiler/src/erl_bifs.erl1
-rw-r--r--lib/debugger/src/dbg_iload.erl1
-rw-r--r--lib/hipe/cerl/erl_bif_types.erl4
-rw-r--r--lib/kernel/doc/src/gen_sctp.xml2
-rw-r--r--lib/sasl/src/release_handler.erl40
-rw-r--r--lib/sasl/src/release_handler_1.erl91
-rw-r--r--lib/sasl/test/release_handler_SUITE.erl135
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/Makefile.src32
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/README16
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.app (renamed from lib/sasl/test/release_handler_SUITE_data/lib/a-1.0/src/a.app)4
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup3
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file0
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl54
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a_sup.erl (renamed from lib/asn1/src/asn1_sup.erl)30
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app7
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl3
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl37
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app7
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup3
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl3
-rw-r--r--lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl37
-rw-r--r--lib/snmp/Makefile5
-rw-r--r--lib/snmp/doc/src/Makefile4
-rw-r--r--lib/snmp/doc/src/notes.xml130
-rw-r--r--lib/snmp/doc/src/snmpc.xml15
-rw-r--r--lib/snmp/doc/src/snmpc_cmd.xml28
-rw-r--r--lib/snmp/doc/src/snmpm.xml40
-rw-r--r--lib/snmp/src/agent/snmp_view_based_acm_mib.erl6
-rw-r--r--lib/snmp/src/agent/snmpa_agent.erl2
-rw-r--r--lib/snmp/src/agent/snmpa_conf.erl9
-rw-r--r--lib/snmp/src/agent/snmpa_mpd.erl104
-rw-r--r--lib/snmp/src/app/snmp.appup.src226
-rw-r--r--lib/snmp/src/compile/Makefile11
-rw-r--r--lib/snmp/src/compile/snmpc.erl9
-rw-r--r--lib/snmp/src/compile/snmpc.src14
-rw-r--r--lib/snmp/src/compile/snmpc_lib.hrl15
-rw-r--r--lib/snmp/src/manager/snmpm.erl8
-rw-r--r--lib/snmp/src/manager/snmpm_config.erl515
-rw-r--r--lib/snmp/src/manager/snmpm_mpd.erl47
-rw-r--r--lib/snmp/src/manager/snmpm_net_if.erl31
-rw-r--r--lib/snmp/src/manager/snmpm_server.erl77
-rw-r--r--lib/snmp/src/misc/snmp_conf.erl83
-rw-r--r--lib/snmp/src/misc/snmp_config.erl10
-rw-r--r--lib/snmp/test/snmp_compiler_test.erl47
-rw-r--r--lib/snmp/test/snmp_manager_test.erl185
-rw-r--r--lib/snmp/vsn.mk2
-rw-r--r--lib/ssl/test/ssl_basic_SUITE.erl50
-rw-r--r--lib/ssl/test/ssl_session_cache_SUITE.erl21
-rw-r--r--lib/stdlib/src/erl_internal.erl2
-rw-r--r--lib/stdlib/src/erl_scan.erl7
-rw-r--r--lib/stdlib/src/io_lib.erl4
-rw-r--r--lib/stdlib/src/io_lib_fread.erl64
-rw-r--r--lib/stdlib/src/otp_internal.erl6
-rw-r--r--lib/stdlib/src/queue.erl127
-rw-r--r--lib/stdlib/test/erl_scan_SUITE.erl4
-rw-r--r--lib/stdlib/test/ets_SUITE.erl76
-rw-r--r--lib/stdlib/test/io_SUITE.erl30
-rw-r--r--lib/stdlib/test/supervisor_SUITE.erl18
-rw-r--r--lib/tools/doc/src/xref.xml8
216 files changed, 25569 insertions, 7135 deletions
diff --git a/INSTALL.md b/INSTALL.md
index 1061c5187a..a19e35b01f 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -273,12 +273,6 @@ Some of the available `configure` options are:
* `--with-ssl=PATH` - Specify location of OpenSSL include and lib
* `--{with,without}-ssl` - OpenSSL (without implies that the `crypto`,
`ssh`, and `ssl` won't be built)
-* `--enable-ethread-pre-pentium4-compatibility` - Enable compatibility with
- x86 processors before pentium 4 (back to 486) in the ethread library. If
- not passed the ethread library (part of the runtime system) will use
- instructions that first appeared on the pentium 4 processor when building
- for x86. This option will be automatically enabled if required on the
- build machine.
* `--with-libatomic_ops=PATH` - Use the `libatomic_ops` library for atomic
memory accesses. If `configure` should inform you about no native atomic
implementation available, you typically want to try using the
@@ -726,7 +720,7 @@ Copyright and License
%CopyrightBegin%
-Copyright Ericsson AB 1998-2010. All Rights Reserved.
+Copyright Ericsson AB 1998-2011. All Rights Reserved.
The contents of this file are subject to the Erlang Public License,
Version 1.1, (the "License"); you may not use this file except in
diff --git a/configure.in b/configure.in
index 36b33ec399..c8d4561a44 100644
--- a/configure.in
+++ b/configure.in
@@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl %CopyrightBegin%
dnl
-dnl Copyright Ericsson AB 1998-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1998-2011. All Rights Reserved.
dnl
dnl The contents of this file are subject to the Erlang Public License,
dnl Version 1.1, (the "License"); you may not use this file except in
@@ -306,10 +306,6 @@ AS_HELP_STRING([--enable-m32-build],
esac
],enable_m32_build=no)
-AC_ARG_ENABLE(ethread-pre-pentium4-compatibility,
- AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility],
- [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]))
-
AC_ARG_WITH(libatomic_ops,
AS_HELP_STRING([--with-libatomic_ops=PATH],
[specify and prefer usage of libatomic_ops in the ethread library]))
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4
index b64380e817..3ebea66d30 100644
--- a/erts/aclocal.m4
+++ b/erts/aclocal.m4
@@ -679,6 +679,55 @@ AC_SUBST(ERTS_INTERNAL_X_LIBS)
])
+AC_DEFUN(ETHR_CHK_SYNC_OP,
+[
+ AC_MSG_CHECKING([for $3-bit $1()])
+ case "$2" in
+ "1") sync_call="$1(&var);";;
+ "2") sync_call="$1(&var, ($4) 0);";;
+ "3") sync_call="$1(&var, ($4) 0, ($4) 0);";;
+ esac
+ have_sync_op=no
+ AC_TRY_LINK([],
+ [
+ $4 res;
+ volatile $4 var;
+ res = $sync_call
+ ],
+ [have_sync_op=yes])
+ test $have_sync_op = yes && $5
+ AC_MSG_RESULT([$have_sync_op])
+])
+
+AC_DEFUN(ETHR_CHK_INTERLOCKED,
+[
+ ilckd="$1"
+ AC_MSG_CHECKING([for ${ilckd}()])
+ case "$2" in
+ "1") ilckd_call="${ilckd}(var);";;
+ "2") ilckd_call="${ilckd}(var, ($3) 0);";;
+ "3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";;
+ "4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";;
+ esac
+ have_interlocked_op=no
+ AC_TRY_LINK(
+ [
+ #define WIN32_LEAN_AND_MEAN
+ #include <windows.h>
+ #include <intrin.h>
+ ],
+ [
+ volatile $3 *var;
+ volatile $3 arr[2];
+
+ $ilckd_call
+ return 0;
+ ],
+ [have_interlocked_op=yes])
+ test $have_interlocked_op = yes && $4
+ AC_MSG_RESULT([$have_interlocked_op])
+])
+
dnl ----------------------------------------------------------------------
dnl
dnl ERL_FIND_ETHR_LIB
@@ -750,121 +799,41 @@ case "$THR_LIB_NAME" in
AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads])
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedCompareExchange64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])
-
- AC_CHECK_SIZEOF(void *)
- case "$ac_cv_sizeof_void_p-$have_ilckd" in
- 8-no)
- ethr_have_native_atomics=no
- ethr_have_native_spinlock=no;;
- *)
- ethr_have_native_atomics=yes
- ethr_have_native_spinlock=yes;;
- esac
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedDecrement64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedDecrement64(var);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedIncrement64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedIncrement64(var);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedExchangeAdd64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedExchangeAdd64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedExchange64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedExchange64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedAnd64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedAnd64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedOr64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedOr64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])
-
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()]))
+
+ test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes
;;
pthread)
@@ -1100,35 +1069,51 @@ case "$THR_LIB_NAME" in
AC_MSG_RESULT([$linux_futex])
test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.])
- AC_MSG_CHECKING([for GCC atomic operations])
- ethr_have_gcc_atomic_ops=no
- AC_TRY_LINK([],
- [
- long res;
- volatile long val;
- res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0);
- res = __sync_add_and_fetch(&val, (long) 1);
- res = __sync_sub_and_fetch(&val, (long) 1);
- res = __sync_fetch_and_and(&val, (long) 1);
- res = __sync_fetch_and_or(&val, (long) 1);
- ],
- [ethr_have_native_atomics=yes
- ethr_have_gcc_atomic_ops=yes])
- AC_MSG_RESULT([$ethr_have_gcc_atomic_ops])
- test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations])
+ AC_CHECK_SIZEOF(int)
+ AC_CHECK_SIZEOF(long)
+ AC_CHECK_SIZEOF(long long)
+ AC_CHECK_SIZEOF(__int128_t)
+
+ if test "$ac_cv_sizeof_int" = "4"; then
+ int32="int"
+ elif test "$ac_cv_sizeof_long" = "4"; then
+ int32="long"
+ elif test "$ac_cv_sizeof_long_long" = "4"; then
+ int32="long long"
+ else
+ AC_MSG_ERROR([No 32-bit type found])
+ fi
- case "$host_cpu" in
- sun4u | sparc64 | sun4v)
- ethr_have_native_atomics=yes;;
- i86pc | i*86 | x86_64 | amd64)
- ethr_have_native_atomics=yes;;
- macppc | ppc | "Power Macintosh")
- ethr_have_native_atomics=yes;;
- tile)
- ethr_have_native_atomics=yes;;
- *)
- ;;
- esac
+ if test "$ac_cv_sizeof_int" = "8"; then
+ int64="int"
+ elif test "$ac_cv_sizeof_long" = "8"; then
+ int64="long"
+ elif test "$ac_cv_sizeof_long_long" = "8"; then
+ int64="long long"
+ else
+ AC_MSG_ERROR([No 64-bit type found])
+ fi
+
+ int128=no
+ if test "$ac_cv_sizeof___int128_t" = "16"; then
+ int128="__int128_t"
+ fi
+
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers]))
+ test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers]))
+
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers]))
+ test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers]))
+
+ if test $int128 != no; then
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers]))
+ fi
AC_MSG_CHECKING([for a usable libatomic_ops implementation])
case "x$with_libatomic_ops" in
@@ -1175,6 +1160,34 @@ case "$THR_LIB_NAME" in
AC_MSG_ERROR([No usable libatomic_ops implementation found])
fi
+ case "$host_cpu" in
+ sparc | sun4u | sparc64 | sun4v)
+ case "$with_sparc_memory_order" in
+ "TSO")
+ AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);;
+ "PSO")
+ AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);;
+ "RMO"|"")
+ AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);;
+ *)
+ AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);;
+ esac
+ ethr_have_native_atomics=yes;;
+ i86pc | i*86 | x86_64 | amd64)
+ if test "$enable_x86_out_of_order" = "yes"; then
+ AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized])
+ fi
+ ethr_have_native_atomics=yes;;
+ macppc | ppc | "Power Macintosh")
+ ethr_have_native_atomics=yes;;
+ tile)
+ ethr_have_native_atomics=yes;;
+ *)
+ ;;
+ esac
+
+ test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes
+
dnl Restore LIBS
LIBS=$saved_libs
dnl restore CPPFLAGS
@@ -1210,6 +1223,8 @@ AC_CHECK_SIZEOF(long long)
AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long])
AC_CHECK_SIZEOF(__int64)
AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64])
+AC_CHECK_SIZEOF(__int128_t)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT128_T, $ac_cv_sizeof___int128_t, [Define to the size of __int128_t])
case X$erl_xcomp_bigendian in
@@ -1232,6 +1247,10 @@ AC_ARG_ENABLE(native-ethr-impls,
*) disable_native_ethr_impls=no ;;
esac ], disable_native_ethr_impls=no)
+AC_ARG_ENABLE(x86-out-of-order,
+ AS_HELP_STRING([--enable-x86-out-of-order],
+ [enable x86/x84_64 out of order support (default disabled)]))
+
test "X$disable_native_ethr_impls" = "Xyes" &&
AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
@@ -1250,55 +1269,101 @@ AC_ARG_WITH(libatomic_ops,
AS_HELP_STRING([--with-libatomic_ops=PATH],
[specify and prefer usage of libatomic_ops in the ethread library]))
-AC_ARG_ENABLE(ethread-pre-pentium4-compatibility,
- AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility],
- [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]),
-[
- case "$enable_ethread_pre_pentium4_compatibility" in
- yes|no) ;;
- *) enable_ethread_pre_pentium4_compatibility=check;;
- esac
-],
-[enable_ethread_pre_pentium4_compatibility=check])
-
-test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no
-
-case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in
- check-i86pc | check-i*86)
- AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced])
- AC_RUN_IFELSE([
-#if defined(__GNUC__)
-# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
-# define CHECK_LIBATOMIC_OPS__
-# else
-# define CHECK_GCC_ASM__
-# endif
-#elif defined(ETHR_HAVE_LIBATOMIC_OPS)
-# define CHECK_LIBATOMIC_OPS__
+AC_ARG_WITH(with_sparc_memory_order,
+ AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO],
+ [specify sparc memory order (defaults to RMO)]))
+
+ETHR_X86_SSE2_ASM=no
+case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in
+ yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64)
+ AC_MSG_CHECKING([for gcc sse2 asm support])
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS -msse2"
+ gcc_sse2_asm=no
+ AC_TRY_COMPILE([],
+ [
+ long long x, *y;
+ __asm__ __volatile__("movq %1, %0\n\t" : "=x"(x) : "m"(*y) : "memory");
+ ],
+ [gcc_sse2_asm=yes])
+ CFLAGS="$save_CFLAGS"
+ AC_MSG_RESULT([$gcc_sse2_asm])
+ if test "$gcc_sse2_asm" = "yes"; then
+ AC_DEFINE(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT, 1, [Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements])
+ ETHR_X86_SSE2_ASM=yes
+ fi
+ ;;
+ *)
+ ;;
+esac
+
+case "$GCC-$host_cpu" in
+ yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64)
+ gcc_dw_cmpxchg_asm=no
+ AC_MSG_CHECKING([for gcc double word cmpxchg asm support])
+ AC_TRY_COMPILE([],
+ [
+ char xchgd;
+ long new[2], xchg[2], *p;
+ __asm__ __volatile__(
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "pushl %%ebx\n\t"
+ "movl %8, %%ebx\n\t"
#endif
-#if defined(CHECK_LIBATOMIC_OPS__)
-#include "atomic_ops.h"
+#if ETHR_SIZEOF_PTR == 4
+ "lock; cmpxchg8b %0\n\t"
+#else
+ "lock; cmpxchg16b %0\n\t"
#endif
-int main(void)
-{
-#if defined(CHECK_GCC_ASM__)
- __asm__ __volatile__("mfence" : : : "memory");
-#elif defined(CHECK_LIBATOMIC_OPS__)
- AO_nop_full();
+ "setz %3\n\t"
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "popl %%ebx\n\t"
#endif
- return 0;
-}
+ : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+ : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]),
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "r"(new[0])
+#else
+ "b"(new[0])
+#endif
+ : "cc", "memory");
+
],
- [enable_ethread_pre_pentium4_compatibility=no],
- [enable_ethread_pre_pentium4_compatibility=yes],
- [enable_ethread_pre_pentium4_compatibility=no])
- AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);;
+ [gcc_dw_cmpxchg_asm=yes])
+ if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then
+ AC_TRY_COMPILE([],
+ [
+ char xchgd;
+ long new[2], xchg[2], *p;
+#if !defined(__PIC__) || !__PIC__
+# error nope
+#endif
+ __asm__ __volatile__(
+ "pushl %%ebx\n\t"
+ "movl (%7), %%ebx\n\t"
+ "movl 4(%7), %%ecx\n\t"
+ "lock; cmpxchg8b %0\n\t"
+ "setz %3\n\t"
+ "popl %%ebx\n\t"
+ : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+ : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new)
+ : "cc", "memory");
+
+ ],
+ [gcc_dw_cmpxchg_asm=yes])
+ if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+ AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code])
+ fi
+ fi
+ AC_MSG_RESULT([$gcc_dw_cmpxchg_asm])
+ if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+ AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction])
+ fi;;
*)
;;
esac
-test $enable_ethread_pre_pentium4_compatibility = yes &&
- AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.])
+
AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \
[Define if you have all ethread defines])
@@ -1309,6 +1374,7 @@ AC_SUBST(ETHR_LIB_NAME)
AC_SUBST(ETHR_DEFS)
AC_SUBST(ETHR_THR_LIB_BASE)
AC_SUBST(ETHR_THR_LIB_BASE_DIR)
+AC_SUBST(ETHR_X86_SSE2_ASM)
])
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index cdce4ec0b8..382e446dce 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -822,6 +822,13 @@ typedef enum {
<desc><p>Create an ordinary list containing the elements of array <c>arr</c>
of length <c>cnt</c>. An empty list is returned if <c>cnt</c> is 0.</p></desc>
</func>
+ <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)</nametext></name>
+ <fsummary>Create the reverse list of the list <c>term</c>.</fsummary>
+ <desc><p>Set <c>*list</c> to the reverse list of the list <c>term</c> and return true,
+ or return false if <c>term</c> is not a list. This function should only be used on
+ short lists as a copy will be created of the list which will not be released until after the
+ nif returns.</p></desc>
+ </func>
<func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_long(ErlNifEnv* env, long int i)</nametext></name>
<fsummary>Create an integer term from a long int</fsummary>
<desc><p>Create an integer term from a <c>long int</c>.</p></desc>
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 7cfab0785d..b7d775541f 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -538,15 +538,6 @@ false</pre>
</desc>
</func>
<func>
- <name name="concat_binary" arity="1"/>
- <fsummary>Concatenate a list of binaries (deprecated)</fsummary>
- <desc>
- <p>Do not use; use
- <seealso marker="#list_to_binary/1">list_to_binary/1</seealso>
- instead.</p>
- </desc>
- </func>
- <func>
<name>erlang:crc32(Data) -> integer() >= 0</name>
<fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary>
<type>
diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml
index 51a4a2bca0..90347824d5 100644
--- a/erts/doc/src/erts_alloc.xml
+++ b/erts/doc/src/erts_alloc.xml
@@ -78,14 +78,20 @@
segments are allocated, cached segments are used if possible
instead of creating new segments. This in order to reduce
the number of system calls made.</item>
+ <tag><c>sbmbc_alloc</c></tag>
+ <item>Allocator used by other allocators for allocation of carriers
+ where only small blocks are placed. Currently this allocator is
+ disabled by default.</item>
</taglist>
<p><c>sys_alloc</c> and <c>fix_alloc</c> are always enabled and
cannot be disabled. <c>mseg_alloc</c> is always enabled if it is
available and an allocator that uses it is enabled. All other
allocators can be <seealso marker="#M_e">enabled or disabled</seealso>.
By default all allocators are enabled.
- When an allocator is disabled, <c>sys_alloc</c>
- is used instead of the disabled allocator.</p>
+ When an allocator is disabled, <c>sys_alloc</c> is used instead of
+ the disabled allocator. <c>sbmbc_alloc</c> is an exception. If
+ <c>sbmbc_alloc</c> is disabled, other allocators will not handle
+ small blocks in separate carriers.</p>
<p>The main idea with the <c>erts_alloc</c> library is to separate
memory blocks that are used differently into different memory
areas, and by this achieving less memory fragmentation. By
@@ -103,15 +109,20 @@
following does <em>not</em> apply to them.</p>
<p>An allocator manages multiple areas, called carriers, in which
memory blocks are placed. A carrier is either placed in a
- separate memory segment (allocated via <c>mseg_alloc</c>) or in
- the heap segment (allocated via <c>sys_alloc</c>). Multiblock
+ separate memory segment (allocated via <c>mseg_alloc</c>), in
+ the heap segment (allocated via <c>sys_alloc</c>), or inside
+ another carrier (in case it is a carrier created by
+ <c>sbmbc_alloc</c>). Multiblock
carriers are used for storage of several blocks. Singleblock
carriers are used for storage of one block. Blocks that are
larger than the value of the singleblock carrier threshold
(<seealso marker="#M_sbct">sbct</seealso>) parameter are placed
- in singleblock carriers. Blocks smaller than the value of the
- <c>sbct</c> parameter are placed in multiblock
- carriers. Normally an allocator creates a "main multiblock
+ in singleblock carriers. Blocks that are smaller than the value
+ of the <c>sbct</c> parameter are placed in multiblock
+ carriers. Blocks that are smaller than the small block multiblock
+ carrier threshold (<seealso marker="#M_sbmbct">sbmbct</seealso>)
+ will be placed in multiblock carriers only used for small blocks.
+ Normally an allocator creates a "main multiblock
carrier". Main multiblock carriers are never deallocated. The
size of the main multiblock carrier is determined by the value
of the <seealso marker="#M_mmbcs">mmbcs</seealso> parameter.</p>
@@ -133,8 +144,11 @@
<c>sbct</c> parameter should be larger than the value of the
<c>lmbcs</c> parameter, the allocator may have to create
multiblock carriers that are larger than the value of the
- <c>lmbcs</c> parameter, though. Singleblock carriers allocated
- via <c>mseg_alloc</c> are sized to whole pages.</p>
+ <c>lmbcs</c> parameter, though. The size of multiblock carriers
+ for small blocks is determined by the small block multiblock
+ carrier size (<seealso marker="#M_sbmbcs">sbmbcs</seealso>).
+ Singleblock carriers allocated via <c>mseg_alloc</c> are sized
+ to whole pages.</p>
<p>Sizes of carriers allocated via <c>sys_alloc</c> are
decided based on the value of the <c>sys_alloc</c> carrier size
(<seealso marker="#Muycs">ycs</seealso>) parameter. The size of
@@ -166,6 +180,14 @@
used. The time complexity is proportional to log N, where
N is the number of free blocks.</p>
</item>
+ <tag>Address order first fit</tag>
+ <item>
+ <p>Strategy: Find the block with the lowest address that satisfies the
+ requested block size.</p>
+ <p>Implementation: A balanced binary search tree is
+ used. The time complexity is proportional to log N, where
+ N is the number of free blocks.</p>
+ </item>
<tag>Good fit</tag>
<item>
<p>Strategy: Try to find the best fit, but settle for the best fit
@@ -194,6 +216,11 @@
</taglist>
</section>
+ <note><p>
+ Currently only allocators using the best fit and the address order
+ best fit strategies are able to use "small block multi block carriers".
+ </p></note>
+
<section>
<marker id="flags"></marker>
<title>System Flags Effecting erts_alloc</title>
@@ -215,6 +242,7 @@
the currently present allocators:</p>
<list type="bulleted">
<item><c>B: binary_alloc</c></item>
+ <item><c>C: sbmbc_alloc</c></item>
<item><c>D: std_alloc</c></item>
<item><c>E: ets_alloc</c></item>
<item><c>F: fix_alloc</c></item>
@@ -300,11 +328,11 @@
subsystem identifier, only the specific allocator identified will be
effected:</p>
<taglist>
- <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|gf|af]]></c></marker></tag>
+ <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|aoff|gf|af]]></c></marker></tag>
<item>
Allocation strategy. Valid strategies are <c>bf</c> (best fit),
- <c>aobf</c> (address order best fit), <c>gf</c> (good fit),
- and <c>af</c> (a fit). See
+ <c>aobf</c> (address order best fit), <c>aoff</c> (address order first fit),
+ <c>gf</c> (good fit), and <c>af</c> (a fit). See
<seealso marker="#strategy">the description of allocation strategies</seealso> in "the <c>alloc_util</c> framework" section.</item>
<tag><marker id="M_asbcst"><c><![CDATA[+M<S>asbcst <size>]]></c></marker></tag>
<item>
@@ -395,6 +423,20 @@
threshold will be placed in singleblock carriers. Blocks
smaller than this threshold will be placed in multiblock
carriers.</item>
+ <tag><marker id="M_sbmbcs"><c><![CDATA[+M<S>sbmbcs <size>]]></c></marker></tag>
+ <item>
+ Small block multiblock carrier size (in bytes). Memory blocks smaller
+ than the small block multiblock carrier threshold
+ (<seealso marker="#M_sbmbct">sbmbct</seealso>) will be placed in
+ multiblock carriers used for small blocks only. This parameter
+ determines the size of such carriers.
+ </item>
+ <tag><marker id="M_sbmbct"><c><![CDATA[+M<S>sbmbct <size>]]></c></marker></tag>
+ <item>
+ Small block multiblock carrier threshold (in bytes). Memory blocks
+ smaller than this threshold will be placed in multiblock carriers
+ used for small blocks only.
+ </item>
<tag><marker id="M_smbcs"><c><![CDATA[+M<S>smbcs <size>]]></c></marker></tag>
<item>
Smallest (<c>mseg_alloc</c>) multiblock carrier size (in
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index d9362a2a8f..b658e79378 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -742,7 +742,7 @@ RUN_OBJS = \
$(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \
$(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \
$(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \
- $(OBJDIR)/erl_bif_binary.o
+ $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o
ifeq ($(TARGET),win32)
DRV_OBJS = \
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index b97705ed96..d7c7f117cf 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -75,7 +75,7 @@ void atom_info(int to, void *to_arg)
index_info(to, to_arg, &erts_atom_table);
#ifdef ERTS_ATOM_PUT_OPS_STAT
erts_print(to, to_arg, "atom_put_ops: %ld\n",
- erts_smp_atomic_read(&atom_put_ops));
+ erts_smp_atomic_read_nob(&atom_put_ops));
#endif
if (lock)
@@ -213,7 +213,7 @@ am_atom_put(const char* name, int len)
len = MAX_ATOM_LENGTH;
}
#ifdef ERTS_ATOM_PUT_OPS_STAT
- erts_smp_atomic_inc(&atom_put_ops);
+ erts_smp_atomic_inc_nob(&atom_put_ops);
#endif
a.len = len;
a.name = (byte*)name;
@@ -309,7 +309,7 @@ init_atom_table(void)
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
#ifdef ERTS_ATOM_PUT_OPS_STAT
- erts_smp_atomic_init(&atom_put_ops, 0);
+ erts_smp_atomic_init_nob(&atom_put_ops, 0);
#endif
erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab");
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 31910888d1..773baad01f 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -408,7 +408,7 @@ erts_is_count_break(BeamInstr *pc, Sint *count_ret) {
if (bdc) {
if (count_ret) {
- *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount);
+ *count_ret = (Sint) erts_smp_atomic_read_nob(&bdc->acount);
}
return !0;
}
@@ -958,17 +958,17 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif,
if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
if (count_op == erts_break_stop) {
- count = erts_smp_atomic_read(&bdc->acount);
+ count = erts_smp_atomic_read_nob(&bdc->acount);
if (count >= 0) {
while(1) {
- res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count);
+ res = erts_smp_atomic_cmpxchg_nob(&bdc->acount, -count - 1, count);
if ((res == count) || count < 0) break;
count = res;
}
}
} else {
/* Reset call counter */
- erts_smp_atomic_set(&bdc->acount, 0);
+ erts_smp_atomic_set_nob(&bdc->acount, 0);
}
} else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
@@ -1097,7 +1097,7 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif,
}
} else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
BpDataCount *bdc = (BpDataCount *) bd;
- erts_smp_atomic_init(&bdc->acount, 0);
+ erts_smp_atomic_init_nob(&bdc->acount, 0);
}
if (bif == BREAK_IS_ERL) {
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index bd8a7249a7..2ec5818688 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -165,8 +165,8 @@ do { \
bdc = (BpDataCount *) bdc->next; \
ASSERT(bdc); \
bds[ix] = (BpData *) bdc; \
- count = erts_smp_atomic_read(&bdc->acount); \
- if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \
+ count = erts_smp_atomic_read_nob(&bdc->acount); \
+ if (count >= 0) erts_smp_atomic_inc_nob(&bdc->acount); \
*(instr_result) = bdc->orig_instr; \
} while (0)
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 68b3350d7f..98dde066fc 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -811,7 +811,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
so.min_heap_size = H_MIN_SIZE;
so.min_vheap_size = BIN_VH_MIN_SIZE;
so.priority = PRIORITY_NORMAL;
- so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
+ so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
so.scheduler = 0;
/*
@@ -3417,10 +3417,10 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */
- erts_smp_atomic_set(&erts_dead_ports_ptr,
- (erts_aint_t) (port_buf + erts_max_ports));
+ erts_smp_atomic_set_nob(&erts_dead_ports_ptr,
+ (erts_aint_t) (port_buf + erts_max_ports));
- next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot);
+ next_ss = erts_smp_atomic32_inc_read_relb(&erts_ports_snapshot);
for (i = erts_max_ports-1; i >= 0; i--) {
Port* prt = &erts_port[i];
@@ -3434,8 +3434,8 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
erts_smp_port_state_unlock(prt);
}
- dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr,
- (erts_aint_t) NULL);
+ dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr,
+ (erts_aint_t) NULL);
erts_smp_mtx_unlock(&ports_snapshot_mtx);
ASSERT(pp <= dead_ports);
@@ -3942,8 +3942,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
goto error;
}
nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n);
- oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs,
- (erts_aint32_t) nval);
+ oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs,
+ (erts_aint32_t) nval);
BIF_RET(make_small(oval));
} else if (BIF_ARG_1 == am_min_heap_size) {
int oval = H_MIN_SIZE;
@@ -4286,7 +4286,7 @@ void erts_init_bif(void)
erts_smp_spinlock_init(&make_ref_lock, "make_ref");
erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot");
- erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL);
+ erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
/*
* bif_return_trap/1 is a hidden BIF that bifs that need to
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index b8889e6206..432b3d0780 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -626,7 +626,7 @@ bin_check(void)
erts_printf("%p orig_size: %bpd, norefs = %bpd\n",
bp->val,
bp->val->orig_size,
- erts_smp_atomic_read(&bp->val->refc));
+ erts_smp_atomic_read_nob(&bp->val->refc));
}
}
if (printed) {
@@ -650,7 +650,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
char dumpnamebuf[MAXPATHLEN];
char* dumpname;
- if (ERTS_IS_CRASH_DUMPING)
+ if (ERTS_SOMEONE_IS_CRASH_DUMPING)
return;
/* Wait for all threads to block. If all threads haven't blocked
@@ -667,7 +667,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
/* Allow us to pass certain places without locking... */
#ifdef ERTS_SMP
- erts_smp_atomic_inc(&erts_writing_erl_crash_dump);
+ erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
+ erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
#else
erts_writing_erl_crash_dump = 1;
#endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index b1cdd0660a..ad042ec088 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -128,8 +128,8 @@ delete_cache(ErtsAtomCache *cache)
{
if (cache) {
erts_free(ERTS_ALC_T_DCACHE, (void *) cache);
- ASSERT(erts_smp_atomic_read(&no_caches) > 0);
- erts_smp_atomic_dec(&no_caches);
+ ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0);
+ erts_smp_atomic_dec_nob(&no_caches);
}
}
@@ -147,7 +147,7 @@ create_cache(DistEntry *dep)
dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE,
sizeof(ErtsAtomCache));
- erts_smp_atomic_inc(&no_caches);
+ erts_smp_atomic_inc_nob(&no_caches);
for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) {
cp->in_arr[i] = THE_NON_VALUE;
cp->out_arr[i] = THE_NON_VALUE;
@@ -156,7 +156,7 @@ create_cache(DistEntry *dep)
Uint erts_dist_cache_size(void)
{
- return (Uint) erts_smp_atomic_read(&no_caches)*sizeof(ErtsAtomCache);
+ return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache);
}
static ErtsProcList *
@@ -444,7 +444,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
ErtsMonitor *monitors;
Uint32 flags;
- erts_smp_atomic_set(&dep->dist_cmd_scheduled, 1);
+ erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
erts_smp_de_rwlock(dep);
ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid)
@@ -510,7 +510,7 @@ void init_dist(void)
{
init_nodes_monitors();
- erts_smp_atomic_init(&no_caches, 0);
+ erts_smp_atomic_init_nob(&no_caches, 0);
/* Lookup/Install all references to trap functions */
dsend2_trap = trap_function(am_dsend,2);
@@ -596,7 +596,7 @@ static void clear_dist_entry(DistEntry *dep)
suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
erts_smp_mtx_unlock(&dep->qlock);
- erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0);
+ erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
dep->send = NULL;
erts_smp_de_rwunlock(dep);
@@ -1775,7 +1775,7 @@ erts_dist_command(Port *prt, int reds_limit)
erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
removed if port command fails */
- erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 0);
+ erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
erts_smp_de_rlock(dep);
flags = dep->flags;
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index 695a4fc3fe..845151c895 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -203,7 +203,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
id = dep->cid;
}
- if (!erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 1)) {
+ if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) {
(void) erts_port_task_schedule(id,
&dep->dist_cmd,
ERTS_PORT_TASK_DIST_CMD,
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index e8b594bb47..d397cd8848 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -43,9 +43,9 @@
/* Prototypes of callback functions */
static Block_t * get_free_block (Allctr_t *, Uint,
- Block_t *, Uint);
-static void link_free_block (Allctr_t *, Block_t *);
-static void unlink_free_block (Allctr_t *, Block_t *);
+ Block_t *, Uint, Uint32);
+static void link_free_block (Allctr_t *, Block_t *, Uint32);
+static void unlink_free_block (Allctr_t *, Block_t *, Uint32);
static Eterm info_options (Allctr_t *, char *, int *,
@@ -72,6 +72,8 @@ erts_afalc_start(AFAllctr_t *afallctr,
is a struct). */
Allctr_t *allctr = (Allctr_t *) afallctr;
+ init->sbmbct = 0; /* Small mbc not supported by afit */
+
sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t));
allctr->mbc_header_size = sizeof(Carrier_t);
@@ -105,7 +107,8 @@ erts_afalc_start(AFAllctr_t *afallctr,
}
static Block_t *
-get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size)
+get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size,
+ Uint32 flags)
{
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
@@ -123,7 +126,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size)
}
static void
-link_free_block(Allctr_t *allctr, Block_t *block)
+link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
@@ -144,7 +147,7 @@ link_free_block(Allctr_t *allctr, Block_t *block)
}
static void
-unlink_free_block(Allctr_t *allctr, Block_t *block)
+unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index cda404af5e..bbc8a445a7 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -50,6 +50,9 @@
#include "erl_bestfit_alloc.h"
#define GET_ERL_AF_ALLOC_IMPL
#include "erl_afit_alloc.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16
@@ -85,15 +88,19 @@ typedef union {
char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
AFAllctr_t afa;
char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
+ AOFFAllctr_t aoffa;
+ char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
} ErtsAllocatorState_t;
-static ErtsAllocatorState_t sl_alloc_state;
+static ErtsAllocatorState_t sbmbc_alloc_state;
static ErtsAllocatorState_t std_alloc_state;
static ErtsAllocatorState_t ll_alloc_state;
#if HALFWORD_HEAP
-static ErtsAllocatorState_t std_alloc_low_state;
-static ErtsAllocatorState_t ll_alloc_low_state;
+static ErtsAllocatorState_t sbmbc_low_alloc_state;
+static ErtsAllocatorState_t std_low_alloc_state;
+static ErtsAllocatorState_t ll_low_alloc_state;
#endif
+static ErtsAllocatorState_t sl_alloc_state;
static ErtsAllocatorState_t temp_alloc_state;
static ErtsAllocatorState_t eheap_alloc_state;
static ErtsAllocatorState_t binary_alloc_state;
@@ -120,7 +127,8 @@ static void *fix_core_alloc(Uint size)
enum allctr_type {
GOODFIT,
BESTFIT,
- AFIT
+ AFIT,
+ AOFIRSTFIT
};
struct au_init {
@@ -132,6 +140,7 @@ struct au_init {
GFAllctrInit_t gf;
BFAllctrInit_t bf;
AFAllctrInit_t af;
+ AOFFAllctrInit_t aoff;
} init;
struct {
int mmbcs;
@@ -145,7 +154,8 @@ struct au_init {
ERTS_DEFAULT_ALLCTR_INIT, \
ERTS_DEFAULT_GF_ALLCTR_INIT, \
ERTS_DEFAULT_BF_ALLCTR_INIT, \
- ERTS_DEFAULT_AF_ALLCTR_INIT \
+ ERTS_DEFAULT_AF_ALLCTR_INIT, \
+ ERTS_DEFAULT_AOFF_ALLCTR_INIT \
}
typedef struct {
@@ -162,6 +172,7 @@ typedef struct {
char *mtrace;
char *nodename;
} instr;
+ struct au_init sbmbc_alloc;
struct au_init sl_alloc;
struct au_init std_alloc;
struct au_init ll_alloc;
@@ -171,8 +182,9 @@ typedef struct {
struct au_init ets_alloc;
struct au_init driver_alloc;
#if HALFWORD_HEAP
- struct au_init std_alloc_low;
- struct au_init ll_alloc_low;
+ struct au_init sbmbc_low_alloc;
+ struct au_init std_low_alloc;
+ struct au_init ll_low_alloc;
#endif
} erts_alc_hndl_args_init_t;
@@ -185,6 +197,34 @@ do { \
} while (0)
static void
+set_default_sbmbc_alloc_opts(struct au_init *ip)
+{
+ SET_DEFAULT_ALLOC_OPTS(ip);
+ ip->enable = 0;
+ ip->thr_spec = 0;
+ ip->atype = BESTFIT;
+ ip->init.bf.ao = 1;
+ ip->init.util.ramv = 0;
+ ip->init.util.mmsbc = 0;
+ ip->init.util.mmmbc = 500;
+ ip->init.util.sbct = ~((UWord) 0);
+ ip->init.util.name_prefix = "sbmbc_";
+ ip->init.util.alloc_no = ERTS_ALC_A_SBMBC;
+#ifndef SMALL_MEMORY
+ ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */
+#else
+ ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */
+#endif
+ ip->init.util.ts = ERTS_ALC_MTA_SBMBC;
+ ip->init.util.asbcst = 0;
+ ip->init.util.rsbcst = 0;
+ ip->init.util.rsbcmt = 0;
+ ip->init.util.rmbcmt = 0;
+ ip->init.util.sbmbct = 0;
+ ip->init.util.sbmbcs = 0;
+}
+
+static void
set_default_sl_alloc_opts(struct au_init *ip)
{
SET_DEFAULT_ALLOC_OPTS(ip);
@@ -202,6 +242,7 @@ set_default_sl_alloc_opts(struct au_init *ip)
ip->init.util.ts = ERTS_ALC_MTA_SHORT_LIVED;
ip->init.util.rsbcst = 80;
#if HALFWORD_HEAP
+ ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
@@ -249,6 +290,8 @@ set_default_ll_alloc_opts(struct au_init *ip)
ip->init.util.rsbcst = 0;
ip->init.util.rsbcmt = 0;
ip->init.util.rmbcmt = 0;
+ ip->init.util.sbmbct = 0;
+ ip->init.util.sbmbcs = 0;
}
static void
@@ -269,6 +312,7 @@ set_default_temp_alloc_opts(struct au_init *ip)
ip->init.util.rsbcst = 90;
ip->init.util.rmbcmt = 100;
#if HALFWORD_HEAP
+ ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
}
@@ -291,6 +335,7 @@ set_default_eheap_alloc_opts(struct au_init *ip)
ip->init.util.ts = ERTS_ALC_MTA_EHEAP;
ip->init.util.rsbcst = 50;
#if HALFWORD_HEAP
+ ip->init.util.force = 1;
ip->init.util.low_mem = 1;
#endif
}
@@ -436,10 +481,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
hdbg_init();
#endif
+ erts_have_sbmbc_alloc = 0;
+
erts_sys_alloc_init();
init_thr_ix(erts_no_schedulers);
erts_init_utils_mem();
+ set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
set_default_sl_alloc_opts(&init.sl_alloc);
set_default_std_alloc_opts(&init.std_alloc);
set_default_ll_alloc_opts(&init.ll_alloc);
@@ -453,6 +501,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
handle_args(argc, argv, &init);
if (erts_no_schedulers <= 1) {
+ init.sbmbc_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
init.ll_alloc.thr_spec = 0;
@@ -464,6 +513,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
if (init.erts_alloc_config) {
/* Adjust flags that erts_alloc_config won't like */
+ init.sbmbc_alloc.thr_spec = 0;
init.temp_alloc.thr_spec = 0;
init.sl_alloc.thr_spec = 0;
init.std_alloc.thr_spec = 0;
@@ -480,6 +530,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
init.temp_alloc.thr_spec = erts_no_schedulers;
/* Others must use thread preferred interface */
+ adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
adjust_tpref(&init.sl_alloc, erts_no_schedulers);
adjust_tpref(&init.std_alloc, erts_no_schedulers);
adjust_tpref(&init.ll_alloc, erts_no_schedulers);
@@ -497,6 +548,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
* The following allocators cannot be run with afit strategy.
* Make sure they don't...
*/
+ refuse_af_strategy(&init.sbmbc_alloc);
refuse_af_strategy(&init.sl_alloc);
refuse_af_strategy(&init.std_alloc);
refuse_af_strategy(&init.ll_alloc);
@@ -518,6 +570,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_afalc_init();
erts_bfalc_init();
erts_gfalc_init();
+ erts_aoffalc_init();
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
erts_allctrs[i].alloc = NULL;
@@ -551,19 +604,30 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
#if HALFWORD_HEAP
/* Init low memory variants by cloning */
- init.std_alloc_low = init.std_alloc;
- init.std_alloc_low.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
- init.std_alloc_low.init.util.low_mem = 1;
-
- init.ll_alloc_low = init.ll_alloc;
- init.ll_alloc_low.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
- init.ll_alloc_low.init.util.low_mem = 1;
-
- set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_alloc_low);
- set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_alloc_low);
+ init.sbmbc_low_alloc = init.sbmbc_alloc;
+ init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
+ init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
+ init.sbmbc_low_alloc.init.util.low_mem = 1;
+
+ init.std_low_alloc = init.std_alloc;
+ init.std_low_alloc.init.util.name_prefix = "std_low_";
+ init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
+ init.std_low_alloc.init.util.force = 1;
+ init.std_low_alloc.init.util.low_mem = 1;
+
+ init.ll_low_alloc = init.ll_alloc;
+ init.ll_low_alloc.init.util.name_prefix = "ll_low_";
+ init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
+ init.ll_low_alloc.init.util.force = 1;
+ init.ll_low_alloc.init.util.low_mem = 1;
+
+ set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc);
+ set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc);
+ set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc);
#endif /* HALFWORD */
set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
+ set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc);
set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc);
@@ -593,6 +657,20 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
+ /* sbmbc_alloc() needs to be started first */
+ start_au_allocator(ERTS_ALC_A_SBMBC,
+ &init.sbmbc_alloc,
+ &sbmbc_alloc_state);
+#if HALFWORD_HEAP
+ start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
+ &init.sbmbc_low_alloc,
+ &sbmbc_low_alloc_state);
+ erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
+ && init.sbmbc_low_alloc.enable);
+#else
+ erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
+#endif
+
start_au_allocator(ERTS_ALC_A_TEMPORARY,
&init.temp_alloc,
&temp_alloc_state);
@@ -610,11 +688,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
&ll_alloc_state);
#if HALFWORD_HEAP
start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
- &init.ll_alloc_low,
- &ll_alloc_low_state);
+ &init.ll_low_alloc,
+ &ll_low_alloc_state);
start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
- &init.std_alloc_low,
- &std_alloc_low_state);
+ &init.std_low_alloc,
+ &std_low_alloc_state);
#endif
start_au_allocator(ERTS_ALC_A_EHEAP,
&init.eheap_alloc,
@@ -680,14 +758,11 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
-#if HALFWORD_HEAP
- /* If halfword heap, silently ignore any disabling of internal
- * allocators for low memory
+ /*
+ * Some allocators are forced on if halfword heap is used.
*/
- if (init->init.util.low_mem) {
+ if (init->init.util.force)
init->enable = 1;
- }
-#endif
if (!init->enable) {
af->alloc = erts_sys_alloc;
@@ -837,6 +912,12 @@ start_au_allocator(ErtsAlcType_t alctr_n,
&init->init.af,
&init->init.util);
break;
+ case AOFIRSTFIT:
+ as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
+ &init->init.aoff,
+ &init->init.util);
+ break;
+
default:
as = NULL;
ASSERT(0);
@@ -947,6 +1028,20 @@ get_kb_value(char *param_end, char** argv, int* ip)
}
static Uint
+get_byte_value(char *param_end, char** argv, int* ip)
+{
+ Sint tmp;
+ char *rest;
+ char *param = argv[*ip]+1;
+ char *value = get_value(param_end, argv, ip);
+ errno = 0;
+ tmp = (Sint) strtol(value, &rest, 10);
+ if (errno != 0 || rest == value || tmp < 0)
+ bad_value(param, param_end, value);
+ return (Uint) tmp;
+}
+
+static Uint
get_amount_value(char *param_end, char** argv, int* ip)
{
Sint tmp;
@@ -1017,6 +1112,9 @@ handle_au_arg(struct au_init *auip,
else if (strcmp("af", alg) == 0) {
auip->atype = AFIT;
}
+ else if (strcmp("aoff", alg) == 0) {
+ auip->atype = AOFIRSTFIT;
+ }
else {
bad_value(param, sub_param + 1, alg);
}
@@ -1085,6 +1183,12 @@ handle_au_arg(struct au_init *auip,
if(has_prefix("sbct", sub_param)) {
auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
}
+ else if (has_prefix("sbmbcs", sub_param)) {
+ auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip);
+ }
+ else if (has_prefix("sbmbct", sub_param)) {
+ auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip);
+ }
else if (has_prefix("smbcs", sub_param)) {
auip->default_.smbcs = 0;
auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
@@ -1123,6 +1227,7 @@ static void
handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
{
struct au_init *aui[] = {
+ &init->sbmbc_alloc,
&init->binary_alloc,
&init->std_alloc,
&init->ets_alloc,
@@ -1150,6 +1255,9 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case 'B':
handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
break;
+ case 'C':
+ handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
+ break;
case 'D':
handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
break;
@@ -1856,12 +1964,16 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
return am_badarg;
}
- /* All alloc_util allocators *have* to be enabled */
+ /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
switch (ai) {
case ERTS_ALC_A_SYSTEM:
case ERTS_ALC_A_FIXED_SIZE:
+ case ERTS_ALC_A_SBMBC:
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_SBMBC_LOW:
+#endif
break;
default:
if (!erts_allctrs_info[ai].enabled
@@ -1901,6 +2013,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
* Often not thread safe and usually never
* contain any allocated memory.
*/
+ case ERTS_ALC_A_SBMBC:
+ /* Included in other allocators */
+#if HALFWORD_HEAP
+ case ERTS_ALC_A_SBMBC_LOW:
+ /* Included in other allocators */
+#endif
continue;
case ERTS_ALC_A_EHEAP:
save = &size.processes;
@@ -2882,6 +3000,7 @@ unsigned long erts_alc_test(unsigned long op,
case 0x2: return erts_bfalc_test(op, a1, a2);
case 0x3: return erts_afalc_test(op, a1, a2);
case 0x4: return erts_mseg_test(op, a1, a2, a3);
+ case 0x5: return erts_aoffalc_test(op, a1, a2);
case 0xf:
switch (op) {
case 0xf00:
@@ -2925,6 +3044,7 @@ unsigned long erts_alc_test(unsigned long op,
init.atype = GOODFIT;
init.init.util.name_prefix = (char *) a1;
init.init.util.ts = a2 ? 1 : 0;
+ init.init.util.sbmbct = 0;
if ((char **) a3) {
char **argv = (char **) a3;
@@ -2960,6 +3080,14 @@ unsigned long erts_alc_test(unsigned long op,
&init.init.af,
&init.init.util);
break;
+ case AOFIRSTFIT:
+ allctr = erts_aoffalc_start((AOFFAllctr_t *)
+ erts_alloc(ERTS_ALC_T_UNDEF,
+ sizeof(AOFFAllctr_t)),
+ &init.init.aoff,
+ &init.init.util);
+ break;
+
default:
ASSERT(0);
allctr = NULL;
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index ce792d4d17..c35a60da22 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -99,6 +99,14 @@ unsigned long erts_alc_test(unsigned long,
#define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000)
+#if HALFWORD_HEAP
+#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \
+ ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW)
+#else
+#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \
+ ((NO) == ERTS_ALC_A_SBMBC)
+#endif
+
typedef struct {
int alloc_util;
int enabled;
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index c6cc0e1fac..eda0831441 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -65,6 +65,11 @@
allocator SYSTEM true sys_alloc
+allocator SBMBC true sbmbc_alloc
++if halfword
+allocator SBMBC_LOW true sbmbc_low_alloc
++endif
+
+if smp
allocator TEMPORARY true temp_alloc
@@ -76,8 +81,8 @@ allocator ETS true ets_alloc
allocator FIXED_SIZE true fix_alloc
+if halfword
-allocator LONG_LIVED_LOW true ll_alloc_low
-allocator STANDARD_LOW true std_alloc_low
+allocator LONG_LIVED_LOW true ll_low_alloc
+allocator STANDARD_LOW true std_low_alloc
+endif
+else # Non smp build
@@ -91,8 +96,8 @@ allocator ETS false ets_alloc
allocator FIXED_SIZE false fix_alloc
+if halfword
-allocator LONG_LIVED_LOW false ll_alloc_low
-allocator STANDARD_LOW false std_alloc_low
+allocator LONG_LIVED_LOW false ll_low_alloc
+allocator STANDARD_LOW false std_low_alloc
+endif
+endif
@@ -134,6 +139,7 @@ class SYSTEM system_data
#
# <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION>
+type SBMBC SBMBC SYSTEM small_block_mbc
type PROC FIXED_SIZE PROCESSES proc
type ATOM FIXED_SIZE ATOM atom_entry
type MODULE FIXED_SIZE CODE module_entry
@@ -330,6 +336,7 @@ type SSB SHORT_LIVED PROCESSES ssb
+if halfword
+type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low
type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes
type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh
type NLINK_LH STANDARD_LOW PROCESSES nlink_lh
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index cc04ef65bf..d51ed0c36d 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -66,6 +66,7 @@
static int atoms_initialized = 0;
static int initialized = 0;
+int erts_have_sbmbc_alloc;
#if HAVE_ERTS_MSEG
@@ -85,8 +86,6 @@ static int initialized = 0;
#undef ASSERT
#define ASSERT ASSERT_EXPR
-#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((UWord) 1)
-
#if 0
/* Can be useful for debugging */
#define MBC_REALLOC_ALWAYS_MOVES
@@ -275,14 +274,26 @@ static void check_blk_carrier(Allctr_t *, Block_t *);
#ifdef DEBUG
#define DEBUG_CHECK_CARRIER_NO_SZ(AP) \
- ASSERT(((AP)->sbcs.curr_mseg.no && (AP)->sbcs.curr_mseg.size) \
- || (!(AP)->sbcs.curr_mseg.no && !(AP)->sbcs.curr_mseg.size));\
- ASSERT(((AP)->sbcs.curr_sys_alloc.no && (AP)->sbcs.curr_sys_alloc.size)\
- || (!(AP)->sbcs.curr_sys_alloc.no && !(AP)->sbcs.curr_sys_alloc.size));\
- ASSERT(((AP)->mbcs.curr_mseg.no && (AP)->mbcs.curr_mseg.size) \
- || (!(AP)->mbcs.curr_mseg.no && !(AP)->mbcs.curr_mseg.size));\
- ASSERT(((AP)->mbcs.curr_sys_alloc.no && (AP)->mbcs.curr_sys_alloc.size)\
- || (!(AP)->mbcs.curr_sys_alloc.no && !(AP)->mbcs.curr_sys_alloc.size))
+ ASSERT(((AP)->sbcs.curr.norm.mseg.no \
+ && (AP)->sbcs.curr.norm.mseg.size) \
+ || (!(AP)->sbcs.curr.norm.mseg.no \
+ && !(AP)->sbcs.curr.norm.mseg.size)); \
+ ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no \
+ && (AP)->sbcs.curr.norm.sys_alloc.size) \
+ || (!(AP)->sbcs.curr.norm.sys_alloc.no \
+ && !(AP)->sbcs.curr.norm.sys_alloc.size)); \
+ ASSERT(((AP)->mbcs.curr.norm.mseg.no \
+ && (AP)->mbcs.curr.norm.mseg.size) \
+ || (!(AP)->mbcs.curr.norm.mseg.no \
+ && !(AP)->mbcs.curr.norm.mseg.size)); \
+ ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \
+ && (AP)->mbcs.curr.norm.sys_alloc.size) \
+ || (!(AP)->mbcs.curr.norm.sys_alloc.no \
+ && !(AP)->mbcs.curr.norm.sys_alloc.size)); \
+ ASSERT(((AP)->sbmbcs.curr.small_block.no \
+ && (AP)->sbmbcs.curr.small_block.size) \
+ || (!(AP)->sbmbcs.curr.small_block.no \
+ && !(AP)->sbmbcs.curr.small_block.size))
#else
#define DEBUG_CHECK_CARRIER_NO_SZ(AP)
@@ -292,27 +303,27 @@ static void check_blk_carrier(Allctr_t *, Block_t *);
(AP)->sbcs.blocks.curr.size += (BSZ); \
if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size) \
(AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size; \
- if ((AP)->sbcs.max.no < ((AP)->sbcs.curr_mseg.no \
- + (AP)->sbcs.curr_sys_alloc.no)) \
- (AP)->sbcs.max.no = ((AP)->sbcs.curr_mseg.no \
- + (AP)->sbcs.curr_sys_alloc.no); \
- if ((AP)->sbcs.max.size < ((AP)->sbcs.curr_mseg.size \
- + (AP)->sbcs.curr_sys_alloc.size)) \
- (AP)->sbcs.max.size = ((AP)->sbcs.curr_mseg.size \
- + (AP)->sbcs.curr_sys_alloc.size)
+ if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no \
+ + (AP)->sbcs.curr.norm.sys_alloc.no)) \
+ (AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no \
+ + (AP)->sbcs.curr.norm.sys_alloc.no); \
+ if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size \
+ + (AP)->sbcs.curr.norm.sys_alloc.size)) \
+ (AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size \
+ + (AP)->sbcs.curr.norm.sys_alloc.size)
#define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ) \
do { \
- (AP)->sbcs.curr_mseg.no++; \
- (AP)->sbcs.curr_mseg.size += (CSZ); \
+ (AP)->sbcs.curr.norm.mseg.no++; \
+ (AP)->sbcs.curr.norm.mseg.size += (CSZ); \
STAT_SBC_ALLOC((AP), (BSZ)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
#define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ) \
do { \
- (AP)->sbcs.curr_sys_alloc.no++; \
- (AP)->sbcs.curr_sys_alloc.size += (CSZ); \
+ (AP)->sbcs.curr.norm.sys_alloc.no++; \
+ (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ); \
STAT_SBC_ALLOC((AP), (BSZ)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
@@ -324,85 +335,111 @@ do { \
#define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ) \
do { \
- ASSERT((AP)->sbcs.curr_mseg.no > 0); \
- (AP)->sbcs.curr_mseg.no--; \
- ASSERT((AP)->sbcs.curr_mseg.size >= (CSZ)); \
- (AP)->sbcs.curr_mseg.size -= (CSZ); \
+ ASSERT((AP)->sbcs.curr.norm.mseg.no > 0); \
+ (AP)->sbcs.curr.norm.mseg.no--; \
+ ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ)); \
+ (AP)->sbcs.curr.norm.mseg.size -= (CSZ); \
STAT_SBC_FREE((AP), (BSZ)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
#define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ) \
do { \
- ASSERT((AP)->sbcs.curr_sys_alloc.no > 0); \
- (AP)->sbcs.curr_sys_alloc.no--; \
- ASSERT((AP)->sbcs.curr_sys_alloc.size >= (CSZ)); \
- (AP)->sbcs.curr_sys_alloc.size -= (CSZ); \
+ ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0); \
+ (AP)->sbcs.curr.norm.sys_alloc.no--; \
+ ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ)); \
+ (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ); \
STAT_SBC_FREE((AP), (BSZ)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
#define STAT_MBC_ALLOC(AP) \
- if ((AP)->mbcs.max.no < ((AP)->mbcs.curr_mseg.no \
- + (AP)->mbcs.curr_sys_alloc.no)) \
- (AP)->mbcs.max.no = ((AP)->mbcs.curr_mseg.no \
- + (AP)->mbcs.curr_sys_alloc.no); \
- if ((AP)->mbcs.max.size < ((AP)->mbcs.curr_mseg.size \
- + (AP)->mbcs.curr_sys_alloc.size)) \
- (AP)->mbcs.max.size = ((AP)->mbcs.curr_mseg.size \
- + (AP)->mbcs.curr_sys_alloc.size)
+ if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no \
+ + (AP)->mbcs.curr.norm.sys_alloc.no)) \
+ (AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no \
+ + (AP)->mbcs.curr.norm.sys_alloc.no); \
+ if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size \
+ + (AP)->mbcs.curr.norm.sys_alloc.size)) \
+ (AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size \
+ + (AP)->mbcs.curr.norm.sys_alloc.size)
+
+#define STAT_SBMBC_ALLOC(AP, CSZ) \
+do { \
+ (AP)->sbmbcs.curr.small_block.no++; \
+ (AP)->sbmbcs.curr.small_block.size += (CSZ); \
+ if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \
+ (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \
+ if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \
+ (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \
+ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
+} while (0)
#define STAT_MSEG_MBC_ALLOC(AP, CSZ) \
do { \
- (AP)->mbcs.curr_mseg.no++; \
- (AP)->mbcs.curr_mseg.size += (CSZ); \
+ (AP)->mbcs.curr.norm.mseg.no++; \
+ (AP)->mbcs.curr.norm.mseg.size += (CSZ); \
STAT_MBC_ALLOC((AP)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
#define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ) \
do { \
- (AP)->mbcs.curr_sys_alloc.no++; \
- (AP)->mbcs.curr_sys_alloc.size += (CSZ); \
+ (AP)->mbcs.curr.norm.sys_alloc.no++; \
+ (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ); \
STAT_MBC_ALLOC((AP)); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
+#define STAT_SBMBC_FREE(AP, CSZ) \
+do { \
+ ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \
+ (AP)->sbmbcs.curr.small_block.no--; \
+ ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \
+ (AP)->sbmbcs.curr.small_block.size -= (CSZ); \
+ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
+} while (0)
+
#define STAT_MSEG_MBC_FREE(AP, CSZ) \
do { \
- ASSERT((AP)->mbcs.curr_mseg.no > 0); \
- (AP)->mbcs.curr_mseg.no--; \
- ASSERT((AP)->mbcs.curr_mseg.size >= (CSZ)); \
- (AP)->mbcs.curr_mseg.size -= (CSZ); \
+ ASSERT((AP)->mbcs.curr.norm.mseg.no > 0); \
+ (AP)->mbcs.curr.norm.mseg.no--; \
+ ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ)); \
+ (AP)->mbcs.curr.norm.mseg.size -= (CSZ); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
#define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ) \
do { \
- ASSERT((AP)->mbcs.curr_sys_alloc.no > 0); \
- (AP)->mbcs.curr_sys_alloc.no--; \
- ASSERT((AP)->mbcs.curr_sys_alloc.size >= (CSZ)); \
- (AP)->mbcs.curr_sys_alloc.size -= (CSZ); \
+ ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0); \
+ (AP)->mbcs.curr.norm.sys_alloc.no--; \
+ ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ)); \
+ (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ); \
DEBUG_CHECK_CARRIER_NO_SZ((AP)); \
} while (0)
-#define STAT_MBC_BLK_ALLOC(AP, BSZ) \
+#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \
do { \
- (AP)->mbcs.blocks.curr.no++; \
- if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \
- (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \
- (AP)->mbcs.blocks.curr.size += (BSZ); \
- if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \
- (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \
+ CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \
+ ? &(AP)->sbmbcs \
+ : &(AP)->mbcs); \
+ cstats__->blocks.curr.no++; \
+ if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \
+ cstats__->blocks.max.no = cstats__->blocks.curr.no; \
+ cstats__->blocks.curr.size += (BSZ); \
+ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \
+ cstats__->blocks.max.size = cstats__->blocks.curr.size; \
} while (0)
-#define STAT_MBC_BLK_FREE(AP, BSZ) \
+#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \
do { \
- ASSERT((AP)->mbcs.blocks.curr.no > 0); \
- (AP)->mbcs.blocks.curr.no--; \
- ASSERT((AP)->mbcs.blocks.curr.size >= (BSZ)); \
- (AP)->mbcs.blocks.curr.size -= (BSZ); \
+ CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \
+ ? &(AP)->sbmbcs \
+ : &(AP)->mbcs); \
+ ASSERT(cstats__->blocks.curr.no > 0); \
+ cstats__->blocks.curr.no--; \
+ ASSERT(cstats__->blocks.curr.size >= (BSZ)); \
+ cstats__->blocks.curr.size -= (BSZ); \
} while (0)
/* Debug stuff... */
@@ -410,7 +447,7 @@ do { \
static UWord carrier_alignment;
#define DEBUG_SAVE_ALIGNMENT(C) \
do { \
- UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t)); \
+ UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\
carrier_alignment = MIN(carrier_alignment, algnmnt__); \
ASSERT(((UWord) (C)) % sizeof(UWord) == 0); \
} while (0)
@@ -524,8 +561,8 @@ static Uint
get_next_mbc_size(Allctr_t *allctr)
{
Uint size;
- int cs = (allctr->mbcs.curr_mseg.no
- + allctr->mbcs.curr_sys_alloc.no
+ int cs = (allctr->mbcs.curr.norm.mseg.no
+ + allctr->mbcs.curr.norm.sys_alloc.no
- (allctr->main_carrier ? 1 : 0));
ASSERT(cs >= 0);
@@ -609,7 +646,8 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
}
}
-
+static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz);
+static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk);
static Block_t *create_carrier(Allctr_t *, Uint, UWord);
static void destroy_carrier(Allctr_t *, Block_t *);
@@ -619,39 +657,57 @@ static void destroy_carrier(Allctr_t *, Block_t *);
* block in a sbc.
*/
static ERTS_INLINE void *
-mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
+mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp)
{
Block_t *blk;
+ Uint get_blk_sz;
+ Uint sbmbct;
ASSERT(size);
ASSERT(size < allctr->sbc_threshold);
- *blk_szp = UMEMSZ2BLKSZ(allctr, size);
+ *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size);
+
+ sbmbct = allctr->sbmbc_threshold;
+ if (sbmbct) {
+ if (get_blk_sz < sbmbct) {
+ *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC;
+ if (get_blk_sz + allctr->min_block_size > sbmbct) {
+ /* Since we use block size to determine if blocks are
+ located in sbmbc or not... */
+ get_blk_sz += allctr->min_block_size;
+ }
+ }
+ }
- blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0);
+ blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp);
-#if HALFWORD_HEAP
if (!blk) {
- blk = create_carrier(allctr, *blk_szp, CFLG_MBC|CFLG_FORCE_MSEG);
- }
+ if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC)
+ blk = create_sbmbc(allctr, get_blk_sz);
+ else {
+#if HALFWORD_HEAP
+ blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG);
#else
- if (!blk) {
- blk = create_carrier(allctr, *blk_szp, CFLG_MBC);
- if (!blk) {
- /* Emergency! We couldn't create the carrier as we wanted.
- Try to place it in a sys_alloced sbc. */
- blk = create_carrier(allctr,
- size,
- CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC);
+ blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
+ if (!blk) {
+ /* Emergency! We couldn't create the carrier as we wanted.
+ Try to place it in a sys_alloced sbc. */
+ blk = create_carrier(allctr,
+ size,
+ (CFLG_SBC
+ | CFLG_FORCE_SIZE
+ | CFLG_FORCE_SYS_ALLOC));
+ }
+#endif
}
}
-#endif
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
if (IS_MBC_BLK(blk)) {
- (*allctr->link_free_block)(allctr, blk);
+ (*allctr->link_free_block)(allctr, blk, *alcu_flgsp);
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, blk);
+ (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp);
}
#endif
@@ -664,7 +720,8 @@ mbc_alloc_finalize(Allctr_t *allctr,
Uint org_blk_sz,
UWord flags,
Uint want_blk_sz,
- int valid_blk_info)
+ int valid_blk_info,
+ Uint32 alcu_flgs)
{
Uint blk_sz;
Uint nxt_blk_sz;
@@ -700,7 +757,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
SET_PREV_BLK_FREE(nxt_nxt_blk);
}
}
- (*allctr->link_free_block)(allctr, nxt_blk);
+ (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_NOT_LAST_BLK(blk));
ASSERT(IS_FREE_BLK(nxt_blk));
@@ -741,7 +798,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
: IS_NOT_LAST_BLK(blk));
}
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+ STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(blk_sz == BLK_SZ(blk));
@@ -761,7 +818,8 @@ mbc_alloc(Allctr_t *allctr, Uint size)
{
Block_t *blk;
Uint blk_sz;
- blk = mbc_alloc_block(allctr, size, &blk_sz);
+ Uint32 alcu_flgs = 0;
+ blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs);
if (!blk)
return NULL;
if (IS_MBC_BLK(blk))
@@ -770,7 +828,8 @@ mbc_alloc(Allctr_t *allctr, Uint size)
BLK_SZ(blk),
GET_BLK_HDR_FLGS(blk),
blk_sz,
- 1);
+ 1,
+ alcu_flgs);
return BLK2UMEM(blk);
}
@@ -779,6 +838,7 @@ mbc_free(Allctr_t *allctr, void *p)
{
Uint is_first_blk;
Uint is_last_blk;
+ Uint32 alcu_flgs = 0;
Uint blk_sz;
Block_t *blk;
Block_t *nxt_blk;
@@ -788,13 +848,15 @@ mbc_free(Allctr_t *allctr, void *p)
blk = UMEM2BLK(p);
blk_sz = BLK_SZ(blk);
+ if (blk_sz < allctr->sbmbc_threshold)
+ alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
ASSERT(IS_MBC_BLK(blk));
ASSERT(blk_sz >= allctr->min_block_size);
HARD_CHECK_BLK_CARRIER(allctr, blk);
- STAT_MBC_BLK_FREE(allctr, blk_sz);
+ STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs);
is_first_blk = IS_FIRST_BLK(blk);
is_last_blk = IS_LAST_BLK(blk);
@@ -802,7 +864,7 @@ mbc_free(Allctr_t *allctr, void *p)
if (!is_first_blk && IS_PREV_BLK_FREE(blk)) {
/* Coalesce with previous block... */
blk = PREV_BLK(blk);
- (*allctr->unlink_free_block)(allctr, blk);
+ (*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
blk_sz += BLK_SZ(blk);
is_first_blk = IS_FIRST_BLK(blk);
@@ -818,7 +880,7 @@ mbc_free(Allctr_t *allctr, void *p)
nxt_blk = NXT_BLK(blk);
if (IS_FREE_BLK(nxt_blk)) {
/* Coalesce with next block... */
- (*allctr->unlink_free_block)(allctr, nxt_blk);
+ (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
blk_sz += BLK_SZ(nxt_blk);
SET_BLK_SZ(blk, blk_sz);
@@ -850,16 +912,20 @@ mbc_free(Allctr_t *allctr, void *p)
if (is_first_blk
&& is_last_blk
- && allctr->main_carrier != FBLK2MBC(allctr, blk))
- destroy_carrier(allctr, blk);
+ && allctr->main_carrier != FBLK2MBC(allctr, blk)) {
+ if (alcu_flgs & ERTS_ALCU_FLG_SBMBC)
+ destroy_sbmbc(allctr, blk);
+ else
+ destroy_carrier(allctr, blk);
+ }
else {
- (*allctr->link_free_block)(allctr, blk);
+ (*allctr->link_free_block)(allctr, blk, alcu_flgs);
HARD_CHECK_BLK_CARRIER(allctr, blk);
}
}
static void *
-mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
+mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
{
void *new_p;
Uint old_blk_sz;
@@ -867,7 +933,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
#ifndef MBC_REALLOC_ALWAYS_MOVES
Block_t *new_blk, *cand_blk;
Uint cand_blk_sz;
- Uint blk_sz;
+ Uint blk_sz, get_blk_sz;
Block_t *nxt_blk;
Uint nxt_blk_sz;
Uint is_last_blk;
@@ -883,10 +949,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
ASSERT(old_blk_sz >= allctr->min_block_size);
#ifdef MBC_REALLOC_ALWAYS_MOVES
- if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+ if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
#else /* !MBC_REALLOC_ALWAYS_MOVES */
- blk_sz = UMEMSZ2BLKSZ(allctr, size);
+ get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
+ if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC)
+ && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) {
+ /* Since we use block size to determine if blocks are
+ located in sbmbc or not... */
+ get_blk_sz = blk_sz + allctr->min_block_size;
+ }
ASSERT(IS_ALLOCED_BLK(blk));
ASSERT(IS_MBC_BLK(blk));
@@ -901,6 +973,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
Uint diff_sz_val = old_blk_sz - blk_sz;
Uint old_blk_sz_val = old_blk_sz;
+ if (get_blk_sz >= old_blk_sz)
+ return p;
+
if (diff_sz_val >= (~((Uint) 0) / 100)) {
/* div both by 128 */
old_blk_sz_val >>= 7;
@@ -909,7 +984,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
/* Avoid fragmentation by moving the block if it is shrunk much */
if (100*diff_sz_val > allctr->mbc_move_threshold*old_blk_sz_val) {
- if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+ if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
cand_blk_sz = old_blk_sz;
@@ -926,9 +1001,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
}
new_blk = (*allctr->get_free_block)(allctr,
- blk_sz,
+ get_blk_sz,
cand_blk,
- cand_blk_sz);
+ cand_blk_sz,
+ alcu_flgs);
if (new_blk || cand_blk != blk)
goto move_into_new_blk;
@@ -952,8 +1028,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
nxt_blk_sz,
SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+ STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
ASSERT(BLK_SZ(blk) >= allctr->min_block_size);
@@ -964,7 +1040,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
if (IS_FREE_BLK(nxt_nxt_blk)) {
/* Coalesce with next free block... */
nxt_blk_sz += BLK_SZ(nxt_nxt_blk);
- (*allctr->unlink_free_block)(allctr, nxt_nxt_blk);
+ (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs);
SET_BLK_SZ(nxt_blk, nxt_blk_sz);
is_last_blk = IS_LAST_BLK(nxt_nxt_blk);
@@ -979,7 +1055,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
}
}
- (*allctr->link_free_block)(allctr, nxt_blk);
+ (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
@@ -1009,12 +1085,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
if (!is_last_blk) {
nxt_blk = NXT_BLK(blk);
nxt_blk_sz = BLK_SZ(nxt_blk);
- if (IS_FREE_BLK(nxt_blk) && blk_sz <= old_blk_sz + nxt_blk_sz) {
+ if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
/* Grow into next block... */
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, nxt_blk);
+ (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
nxt_blk_sz -= blk_sz - old_blk_sz;
is_last_blk = IS_LAST_BLK(nxt_blk);
@@ -1051,13 +1127,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
else
SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
- (*allctr->link_free_block)(allctr, nxt_blk);
+ (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
ASSERT(IS_FREE_BLK(nxt_blk));
}
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
- STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+ STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
+ STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
ASSERT(IS_ALLOCED_BLK(blk));
@@ -1088,7 +1164,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
}
}
- if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+ if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
/* Need to grow in another block */
@@ -1108,7 +1184,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
}
}
- if (cand_blk_sz < blk_sz) {
+ if (cand_blk_sz < get_blk_sz) {
/* We wont fit in cand_blk get a new one */
#endif /* !MBC_REALLOC_ALWAYS_MOVES */
@@ -1127,9 +1203,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
/* We will at least fit in cand_blk */
new_blk = (*allctr->get_free_block)(allctr,
- blk_sz,
+ get_blk_sz,
cand_blk,
- cand_blk_sz);
+ cand_blk_sz,
+ alcu_flgs);
move_into_new_blk:
/*
* new_blk, and cand_blk have to be correctly set
@@ -1142,7 +1219,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
BLK_SZ(new_blk),
GET_BLK_HDR_FLGS(new_blk),
blk_sz,
- 1);
+ 1,
+ alcu_flgs);
new_p = BLK2UMEM(new_blk);
sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
mbc_free(allctr, p);
@@ -1164,7 +1242,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, new_blk); /* prev */
+ (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */
if (is_last_blk)
new_blk_flgs |= LAST_BLK_HDR_FLG;
@@ -1173,7 +1251,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
if (IS_FREE_BLK(nxt_blk)) {
new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
new_blk_sz += BLK_SZ(nxt_blk);
- (*allctr->unlink_free_block)(allctr, nxt_blk);
+ (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
}
}
@@ -1196,9 +1274,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, UWord flgs)
new_blk_sz,
new_blk_flgs,
blk_sz,
- 0);
+ 0,
+ alcu_flgs);
- STAT_MBC_BLK_FREE(allctr, old_blk_sz);
+ STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
return new_p;
}
@@ -1243,6 +1322,100 @@ do { \
#define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ)
#endif
+static Block_t *
+create_sbmbc(Allctr_t *allctr, Uint umem_sz)
+{
+ Block_t *blk;
+ Uint blk_sz;
+ Uint crr_sz = allctr->sbmbc_size;
+ Carrier_t *crr;
+
+#if HALFWORD_HEAP
+ if (allctr->mseg_opt.low_mem)
+ crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz);
+ else
+#endif
+ crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz);
+
+ INC_CC(allctr->calls.sbmbc_alloc);
+ SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC);
+
+ blk = MBC2FBLK(allctr, crr);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+ crr_sz -= sizeof(UWord);
+#endif
+
+ blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
+
+ SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
+ SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ *((Carrier_t **) NXT_BLK(blk)) = crr;
+#endif
+
+ link_carrier(&allctr->sbmbc_list, crr);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+ crr_sz += sizeof(UWord);
+#endif
+
+ STAT_SBMBC_ALLOC(allctr, crr_sz);
+ CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz);
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+ crr_sz -= sizeof(UWord);
+#endif
+ if (allctr->creating_mbc)
+ (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
+
+ DEBUG_SAVE_ALIGNMENT(crr);
+ return blk;
+}
+
+static void
+destroy_sbmbc(Allctr_t *allctr, Block_t *blk)
+{
+ Uint crr_sz;
+ Carrier_t *crr;
+
+ ASSERT(IS_FIRST_BLK(blk));
+
+ ASSERT(IS_MBC_BLK(blk));
+
+ crr = FBLK2MBC(allctr, blk);
+ crr_sz = CARRIER_SZ(crr);
+
+#ifdef DEBUG
+ if (!allctr->stopped) {
+ ASSERT(IS_LAST_BLK(blk));
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
+ HARD_CHECK_BLK_CARRIER(allctr, blk);
+ (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
+#endif
+ }
+#endif
+
+ STAT_SBMBC_FREE(allctr, crr_sz);
+
+ unlink_carrier(&allctr->sbmbc_list, crr);
+ if (allctr->destroying_mbc)
+ (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
+
+ INC_CC(allctr->calls.sbmbc_free);
+
+#if HALFWORD_HEAP
+ if (allctr->mseg_opt.low_mem)
+ erts_free(ERTS_ALC_T_SBMBC_LOW, crr);
+ else
+#endif
+ erts_free(ERTS_ALC_T_SBMBC, crr);
+}
static Block_t *
create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
@@ -1271,11 +1444,11 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
if (erts_mseg_no() >= max_mseg_carriers)
goto try_sys_alloc;
if (flags & CFLG_SBC) {
- if (allctr->sbcs.curr_mseg.no >= allctr->max_mseg_sbcs)
+ if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs)
goto try_sys_alloc;
}
else {
- if (allctr->mbcs.curr_mseg.no >= allctr->max_mseg_mbcs)
+ if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs)
goto try_sys_alloc;
}
@@ -1289,7 +1462,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
if (crr_sz < allctr->mbc_header_size + blk_sz)
crr_sz = allctr->mbc_header_size + blk_sz;
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (sizeof(Unit_t) == sizeof(UWord))
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
crr_sz += sizeof(UWord);
#endif
}
@@ -1330,7 +1503,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
&& bcrr_sz < allctr->smallest_mbc_size)
bcrr_sz = allctr->smallest_mbc_size;
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (sizeof(Unit_t) == sizeof(UWord))
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
bcrr_sz += sizeof(UWord);
#endif
@@ -1385,7 +1558,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
blk = MBC2FBLK(allctr, crr);
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (sizeof(Unit_t) == sizeof(UWord))
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
crr_sz -= sizeof(UWord);
#endif
@@ -1406,16 +1579,16 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
link_carrier(&allctr->mbc_list, crr);
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (sizeof(Unit_t) == sizeof(UWord))
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
crr_sz += sizeof(UWord);
#endif
CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- if (sizeof(Unit_t) == sizeof(UWord))
+ if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
crr_sz -= sizeof(UWord);
#endif
if (allctr->creating_mbc)
- (*allctr->creating_mbc)(allctr, crr);
+ (*allctr->creating_mbc)(allctr, crr, 0);
}
@@ -1595,9 +1768,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
ASSERT(IS_LAST_BLK(blk));
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
- (*allctr->link_free_block)(allctr, blk);
+ (*allctr->link_free_block)(allctr, blk, 0);
HARD_CHECK_BLK_CARRIER(allctr, blk);
- (*allctr->unlink_free_block)(allctr, blk);
+ (*allctr->unlink_free_block)(allctr, blk, 0);
#endif
}
#endif
@@ -1614,7 +1787,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
unlink_carrier(&allctr->mbc_list, crr);
if (allctr->destroying_mbc)
- (*allctr->destroying_mbc)(allctr, crr);
+ (*allctr->destroying_mbc)(allctr, crr, 0);
}
@@ -1658,12 +1831,15 @@ static struct {
Eterm lmbcs;
Eterm smbcs;
Eterm mbcgs;
+ Eterm sbmbcs;
+ Eterm sbmbct;
#if HAVE_ERTS_MSEG
Eterm mmc;
#endif
Eterm ycs;
+ /* Eterm sbmbcs; */
Eterm mbcs;
Eterm sbcs;
Eterm sys_alloc_carriers_size;
@@ -1688,6 +1864,8 @@ static struct {
Eterm mseg_dealloc;
Eterm mseg_realloc;
#endif
+ Eterm sbmbc_alloc;
+ Eterm sbmbc_free;
#ifdef DEBUG
Eterm end_of_atoms;
#endif
@@ -1746,12 +1924,15 @@ init_atoms(Allctr_t *allctr)
AM_INIT(lmbcs);
AM_INIT(smbcs);
AM_INIT(mbcgs);
+ AM_INIT(sbmbcs);
+ AM_INIT(sbmbct);
#if HAVE_ERTS_MSEG
AM_INIT(mmc);
#endif
AM_INIT(ycs);
+ /*AM_INIT(sbmbcs);*/
AM_INIT(mbcs);
AM_INIT(sbcs);
AM_INIT(sys_alloc_carriers_size);
@@ -1776,6 +1957,8 @@ init_atoms(Allctr_t *allctr)
AM_INIT(mseg_dealloc);
AM_INIT(mseg_realloc);
#endif
+ AM_INIT(sbmbc_free);
+ AM_INIT(sbmbc_alloc);
#ifdef DEBUG
for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
@@ -1869,7 +2052,9 @@ sz_info_carriers(Allctr_t *allctr,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
- Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size;
+ Uint curr_size = (cs == &allctr->sbmbcs
+ ? cs->curr.small_block.size
+ : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size);
if (print_to_p) {
int to = *print_to_p;
@@ -1917,8 +2102,17 @@ info_carriers(Allctr_t *allctr,
Uint *szp)
{
Eterm res = THE_NON_VALUE;
- Uint curr_no = cs->curr_mseg.no + cs->curr_sys_alloc.no;
- Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size;
+ Uint curr_no, curr_size;
+ int small_block = cs == &allctr->sbmbcs;
+
+ if (small_block) {
+ curr_no = cs->curr.small_block.no;
+ curr_size = cs->curr.small_block.size;
+ }
+ else {
+ curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
+ curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
+ }
if (print_to_p) {
int to = *print_to_p;
@@ -1944,18 +2138,20 @@ info_carriers(Allctr_t *allctr,
curr_no,
cs->max.no,
cs->max_ever.no);
+ if (!small_block) {
#if HAVE_ERTS_MSEG
- erts_print(to,
- arg,
- "%smseg carriers: %bpu\n",
- prefix,
- cs->curr_mseg.no);
-#endif
- erts_print(to,
- arg,
- "%ssys_alloc carriers: %bpu\n",
- prefix,
- cs->curr_sys_alloc.no);
+ erts_print(to,
+ arg,
+ "%smseg carriers: %bpu\n",
+ prefix,
+ cs->curr.norm.mseg.no);
+#endif
+ erts_print(to,
+ arg,
+ "%ssys_alloc carriers: %bpu\n",
+ prefix,
+ cs->curr.norm.sys_alloc.no);
+ }
erts_print(to,
arg,
"%scarriers size: %beu %bpu %bpu\n",
@@ -1963,43 +2159,49 @@ info_carriers(Allctr_t *allctr,
curr_size,
cs->max.size,
cs->max_ever.size);
+ if (!small_block) {
#if HAVE_ERTS_MSEG
- erts_print(to,
- arg,
- "%smseg carriers size: %bpu\n",
- prefix,
- cs->curr_mseg.size);
-#endif
- erts_print(to,
- arg,
- "%ssys_alloc carriers size: %bpu\n",
- prefix,
- cs->curr_sys_alloc.size);
+ erts_print(to,
+ arg,
+ "%smseg carriers size: %bpu\n",
+ prefix,
+ cs->curr.norm.mseg.size);
+#endif
+ erts_print(to,
+ arg,
+ "%ssys_alloc carriers size: %bpu\n",
+ prefix,
+ cs->curr.norm.sys_alloc.size);
+ }
}
if (hpp || szp) {
res = NIL;
- add_2tup(hpp, szp, &res,
- am.sys_alloc_carriers_size,
- bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.size));
+ if (!small_block) {
+ add_2tup(hpp, szp, &res,
+ am.sys_alloc_carriers_size,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size));
#if HAVE_ERTS_MSEG
- add_2tup(hpp, szp, &res,
- am.mseg_alloc_carriers_size,
- bld_unstable_uint(hpp, szp, cs->curr_mseg.size));
+ add_2tup(hpp, szp, &res,
+ am.mseg_alloc_carriers_size,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size));
#endif
+ }
add_4tup(hpp, szp, &res,
am.carriers_size,
bld_unstable_uint(hpp, szp, curr_size),
bld_unstable_uint(hpp, szp, cs->max.size),
bld_unstable_uint(hpp, szp, cs->max_ever.size));
- add_2tup(hpp, szp, &res,
- am.sys_alloc_carriers,
- bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.no));
+ if (!small_block) {
+ add_2tup(hpp, szp, &res,
+ am.sys_alloc_carriers,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no));
#if HAVE_ERTS_MSEG
- add_2tup(hpp, szp, &res,
- am.mseg_alloc_carriers,
- bld_unstable_uint(hpp, szp, cs->curr_mseg.no));
+ add_2tup(hpp, szp, &res,
+ am.mseg_alloc_carriers,
+ bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no));
#endif
+ }
add_4tup(hpp, szp, &res,
am.carriers,
bld_unstable_uint(hpp, szp, curr_no),
@@ -2077,6 +2279,9 @@ info_calls(Allctr_t *allctr,
PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free);
PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc);
+ PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc);
+ PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free);
+
#if HAVE_ERTS_MSEG
PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc);
PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc);
@@ -2128,6 +2333,14 @@ info_calls(Allctr_t *allctr,
bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no));
#endif
add_3tup(hpp, szp, &res,
+ am.sbmbc_free,
+ bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no),
+ bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no));
+ add_3tup(hpp, szp, &res,
+ am.sbmbc_alloc,
+ bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no),
+ bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no));
+ add_3tup(hpp, szp, &res,
allctr->name.realloc,
bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no),
bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no));
@@ -2191,7 +2404,9 @@ info_options(Allctr_t *allctr,
#endif
"option lmbcs: %beu\n"
"option smbcs: %beu\n"
- "option mbcgs: %beu\n",
+ "option mbcgs: %beu\n"
+ "option sbmbcs: %beu\n"
+ "option sbmbct: %beu\n",
topt,
allctr->ramv ? "true" : "false",
#if HALFWORD_HEAP
@@ -2211,7 +2426,9 @@ info_options(Allctr_t *allctr,
#endif
allctr->largest_mbc_size,
allctr->smallest_mbc_size,
- allctr->mbc_growth_stages);
+ allctr->mbc_growth_stages,
+ allctr->sbmbc_size,
+ allctr->sbmbc_threshold);
}
res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg,
@@ -2219,6 +2436,12 @@ info_options(Allctr_t *allctr,
if (hpp || szp) {
add_2tup(hpp, szp, &res,
+ am.sbmbct,
+ bld_uint(hpp, szp, allctr->sbmbc_threshold));
+ add_2tup(hpp, szp, &res,
+ am.sbmbcs,
+ bld_uint(hpp, szp, allctr->sbmbc_size));
+ add_2tup(hpp, szp, &res,
am.mbcgs,
bld_uint(hpp, szp, allctr->mbc_growth_stages));
add_2tup(hpp, szp, &res,
@@ -2285,10 +2508,10 @@ update_max_ever_values(CarriersStats_t *cs)
static ERTS_INLINE void
reset_max_values(CarriersStats_t *cs)
{
- cs->max.no = cs->curr_mseg.no + cs->curr_sys_alloc.no;
- cs->max.size = cs->curr_mseg.size + cs->curr_sys_alloc.size;
- cs->blocks.max.no = cs->blocks.curr.no;
- cs->blocks.max.size = cs->blocks.curr.size;
+ cs->max.no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
+ cs->max.size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
+ cs->blocks.max.no = cs->blocks.curr.no;
+ cs->blocks.max.size = cs->blocks.curr.size;
}
@@ -2367,7 +2590,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
Uint **hpp,
Uint *szp)
{
- Eterm res, mbcs, sbcs;
+ Eterm res, sbmbcs, mbcs, sbcs;
res = THE_NON_VALUE;
@@ -2389,24 +2612,29 @@ erts_alcu_sz_info(Allctr_t *allctr,
/* Update sbc values not continously updated */
allctr->sbcs.blocks.curr.no
- = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no;
+ = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
+ update_max_ever_values(&allctr->sbmbcs);
update_max_ever_values(&allctr->mbcs);
update_max_ever_values(&allctr->sbcs);
- mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
- print_to_arg, hpp, szp);
- sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
- print_to_arg, hpp, szp);
+ sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
+ print_to_arg, hpp, szp);
+ mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
+ print_to_arg, hpp, szp);
+ sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
+ print_to_arg, hpp, szp);
if (hpp || szp) {
res = NIL;
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
+ add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
}
if (begin_max_period) {
+ reset_max_values(&allctr->sbmbcs);
reset_max_values(&allctr->mbcs);
reset_max_values(&allctr->sbcs);
}
@@ -2428,7 +2656,7 @@ erts_alcu_info(Allctr_t *allctr,
Uint **hpp,
Uint *szp)
{
- Eterm res, sett, mbcs, sbcs, calls;
+ Eterm res, sett, sbmbcs, mbcs, sbcs, calls;
res = THE_NON_VALUE;
@@ -2450,9 +2678,10 @@ erts_alcu_info(Allctr_t *allctr,
/* Update sbc values not continously updated */
allctr->sbcs.blocks.curr.no
- = allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no;
+ = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
+ update_max_ever_values(&allctr->sbmbcs);
update_max_ever_values(&allctr->mbcs);
update_max_ever_values(&allctr->sbcs);
@@ -2464,11 +2693,13 @@ erts_alcu_info(Allctr_t *allctr,
ERTS_ALCU_VSN_STR);
}
- sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
- mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
- print_to_arg, hpp, szp);
- sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
- print_to_arg, hpp, szp);
+ sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
+ sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
+ print_to_arg, hpp, szp);
+ mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
+ print_to_arg, hpp, szp);
+ sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
+ print_to_arg, hpp, szp);
calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp);
if (hpp || szp) {
@@ -2477,6 +2708,7 @@ erts_alcu_info(Allctr_t *allctr,
add_2tup(hpp, szp, &res, am.calls, calls);
add_2tup(hpp, szp, &res, am.sbcs, sbcs);
add_2tup(hpp, szp, &res, am.mbcs, mbcs);
+ add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
add_2tup(hpp, szp, &res, am.options, sett);
add_3tup(hpp, szp, &res,
am.versions,
@@ -2485,6 +2717,7 @@ erts_alcu_info(Allctr_t *allctr,
}
if (begin_max_period) {
+ reset_max_values(&allctr->sbmbcs);
reset_max_values(&allctr->mbcs);
reset_max_values(&allctr->sbcs);
}
@@ -2508,12 +2741,14 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size)
erts_mtx_lock(&allctr->mutex);
#endif
- size->carriers = allctr->mbcs.curr_mseg.size;
- size->carriers += allctr->mbcs.curr_sys_alloc.size;
- size->carriers += allctr->sbcs.curr_mseg.size;
- size->carriers += allctr->sbcs.curr_sys_alloc.size;
+ size->carriers = allctr->mbcs.curr.norm.mseg.size;
+ size->carriers += allctr->mbcs.curr.norm.sys_alloc.size;
+ size->carriers += allctr->sbmbcs.curr.small_block.size;
+ size->carriers += allctr->sbcs.curr.norm.mseg.size;
+ size->carriers += allctr->sbcs.curr.norm.sys_alloc.size;
size->blocks = allctr->mbcs.blocks.curr.size;
+ size->blocks += allctr->sbmbcs.blocks.curr.size;
size->blocks += allctr->sbcs.blocks.curr.size;
#ifdef USE_THREADS
@@ -2725,7 +2960,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
void *extra,
void *p,
Uint size,
- UWord flgs)
+ Uint32 alcu_flgs)
{
Allctr_t *allctr = (Allctr_t *) extra;
Block_t *blk;
@@ -2758,9 +2993,32 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
blk = UMEM2BLK(p);
+ if (allctr->sbmbc_threshold > 0) {
+ Uint old_sz, new_sz, lim;
+ lim = allctr->sbmbc_threshold;
+ old_sz = BLK_SZ(blk);
+ new_sz = UMEMSZ2BLKSZ(allctr, size);
+ if ((old_sz < lim && lim <= new_sz)
+ || (new_sz < lim && lim <= old_sz)) {
+ /* *Need* to move it... */
+
+ INC_CC(allctr->calls.this_realloc);
+ res = do_erts_alcu_alloc(type, extra, size);
+ DEC_CC(allctr->calls.this_alloc);
+
+ sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ));
+
+ do_erts_alcu_free(type, extra, p);
+ DEC_CC(allctr->calls.this_free);
+ return res;
+ }
+ if (old_sz < lim)
+ alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
+ }
+
if (size < allctr->sbc_threshold) {
if (IS_MBC_BLK(blk))
- res = mbc_realloc(allctr, p, size, flgs);
+ res = mbc_realloc(allctr, p, size, alcu_flgs);
else {
Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size;
Uint crr_sz;
@@ -2791,7 +3049,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
if (100*diff_sz_val < allctr->sbc_move_threshold*crr_sz_val)
/* Data won't be copied into a new carrier... */
goto do_carrier_resize;
- else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+ else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
res = mbc_alloc(allctr, size);
@@ -2814,7 +3072,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
#endif
res = new_blk ? BLK2UMEM(new_blk) : NULL;
}
- else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+ else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
return NULL;
else {
#if HALFWORD_HEAP
@@ -3174,6 +3432,26 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->min_block_size = UNIT_CEILING(allctr->min_block_size
+ sizeof(UWord));
+
+ allctr->sbmbc_threshold = init->sbmbct;
+
+ if (!erts_have_sbmbc_alloc
+ || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no))
+ allctr->sbmbc_threshold = 0;
+
+ if (!allctr->sbmbc_threshold)
+ allctr->sbmbc_size = 0;
+ else {
+ Uint min_size;
+ allctr->sbmbc_size = init->sbmbcs;
+ min_size = allctr->sbmbc_threshold;
+ min_size += allctr->min_block_size;
+ min_size += allctr->mbc_header_size;
+ if (allctr->sbmbc_size < min_size)
+ allctr->sbmbc_size = min_size;
+ }
+
+
#if HAVE_ERTS_MSEG
if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100)
allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100;
@@ -3185,12 +3463,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
#ifdef ERTS_ENABLE_LOCK_COUNT
erts_mtx_init_x_opt(&allctr->mutex,
- "alcu_allocator",
- make_small(allctr->alloc_no),
- ERTS_LCNT_LT_ALLOC);
+ ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
+ ? "sbmbc_alloc"
+ : "alcu_allocator",
+ make_small(allctr->alloc_no),
+ ERTS_LCNT_LT_ALLOC);
#else
erts_mtx_init_x(&allctr->mutex,
- "alcu_allocator",
+ ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
+ ? "sbmbc_alloc"
+ : "alcu_allocator",
make_small(allctr->alloc_no));
#endif /*ERTS_ENABLE_LOCK_COUNT*/
@@ -3260,7 +3542,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
if (!blk)
goto error;
- (*allctr->link_free_block)(allctr, blk);
+ (*allctr->link_free_block)(allctr, blk, 0);
HARD_CHECK_BLK_CARRIER(allctr, blk);
@@ -3290,6 +3572,8 @@ erts_alcu_stop(Allctr_t *allctr)
destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first));
while (allctr->mbc_list.first)
destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first));
+ while (allctr->sbmbc_list.first)
+ destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first));
#ifdef USE_THREADS
if (allctr->thread_safe)
@@ -3387,13 +3671,15 @@ erts_alcu_verify_unused(Allctr_t *allctr)
{
UWord no;
- no = allctr->sbcs.curr_mseg.no;
- no += allctr->sbcs.curr_sys_alloc.no;
+ no = allctr->sbcs.curr.norm.mseg.no;
+ no += allctr->sbcs.curr.norm.sys_alloc.no;
no += allctr->mbcs.blocks.curr.no;
+ no += allctr->sbmbcs.blocks.curr.no;
if (no) {
UWord sz = allctr->sbcs.blocks.curr.size;
sz += allctr->mbcs.blocks.curr.size;
+ sz += allctr->sbmbcs.blocks.curr.size;
erl_exit(ERTS_ABORT_EXIT,
"%salloc() used when expected to be unused!\n"
"Total amount of blocks allocated: %bpu\n"
@@ -3492,7 +3778,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
(*allctr->check_block)(allctr, blk, (int) is_free_blk);
if (IS_LAST_BLK(blk)) {
- carrier_end = ((char *) NXT_BLK(blk)) + sizeof(UWord);
+ carrier_end = ((char *) NXT_BLK(blk));
mbc = *((Carrier_t **) NXT_BLK(blk));
prev_blk = NULL;
blk = MBC2FBLK(allctr, mbc);
@@ -3507,9 +3793,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
ASSERT(IS_MB_CARRIER(mbc));
ASSERT((((char *) mbc)
+ allctr->mbc_header_size
- + tot_blk_sz
- + sizeof(UWord)) == carrier_end);
- ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end);
+ + tot_blk_sz) == carrier_end);
+ ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end
+ && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc));
if (allctr->check_mbc)
(*allctr->check_mbc)(allctr, mbc);
@@ -3523,6 +3809,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
cl = &allctr->mbc_list;
}
+#if 0 /* FIXIT sbmbc */
if (cl->first == crr) {
ASSERT(!crr->prev);
}
@@ -3537,6 +3824,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
ASSERT(crr->next);
ASSERT(crr->next->prev == crr);
}
+#endif
}
#endif
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index ddf84c086c..fed4d3dbe6 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -34,6 +34,7 @@ typedef struct {
typedef struct {
char *name_prefix;
ErtsAlcType_t alloc_no;
+ int force;
int ts;
int tspec;
int tpref;
@@ -50,6 +51,8 @@ typedef struct {
UWord lmbcs;
UWord smbcs;
UWord mbcgs;
+ UWord sbmbct;
+ UWord sbmbcs;
} AllctrInit_t;
typedef struct {
@@ -67,6 +70,7 @@ typedef struct {
#define ERTS_DEFAULT_ALLCTR_INIT { \
NULL, \
ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\
+ 0, /* (bool) force: force enabled */\
1, /* (bool) ts: thread safe */\
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
@@ -82,7 +86,9 @@ typedef struct {
10, /* (amount) mmmbc: max mseg mbcs */\
10*1024*1024, /* (bytes) lmbcs: largest mbc size */\
1024*1024, /* (bytes) smbcs: smallest mbc size */\
- 10 /* (amount) mbcgs: mbc growth stages */\
+ 10, /* (amount) mbcgs: mbc growth stages */\
+ 256, /* (bytes) sbmbct: small block mbc threshold */\
+ 8*1024 /* (bytes) sbmbcs: small block mbc size */\
}
#else /* if SMALL_MEMORY */
@@ -95,6 +101,7 @@ typedef struct {
#define ERTS_DEFAULT_ALLCTR_INIT { \
NULL, \
ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\
+ 0, /* (bool) force: force enabled */\
1, /* (bool) ts: thread safe */\
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
@@ -109,7 +116,9 @@ typedef struct {
10, /* (amount) mmmbc: max mseg mbcs */\
1024*1024, /* (bytes) lmbcs: largest mbc size */\
128*1024, /* (bytes) smbcs: smallest mbc size */\
- 10 /* (amount) mbcgs: mbc growth stages */\
+ 10, /* (amount) mbcgs: mbc growth stages */\
+ 256, /* (bytes) sbmbct: small block mbc threshold */\
+ 8*1024 /* (bytes) sbmbcs: small block mbc size */\
}
#endif
@@ -144,6 +153,9 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
#if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
#define ERL_ALLOC_UTIL_IMPL__
+#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0)
+#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1)
+
#ifdef USE_THREADS
#define ERL_THREADS_EMU_INTERNAL__
#include "erl_threads.h"
@@ -188,6 +200,8 @@ void erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
#define CARRIER_SZ(C) \
((C)->chdr & SZ_MASK)
+extern int erts_have_sbmbc_alloc;
+
typedef union {char c[8]; long l; double d;} Unit_t;
typedef struct Carrier_t_ Carrier_t;
@@ -216,8 +230,13 @@ typedef struct {
} StatValues_t;
typedef struct {
- StatValues_t curr_mseg;
- StatValues_t curr_sys_alloc;
+ union {
+ struct {
+ StatValues_t mseg;
+ StatValues_t sys_alloc;
+ } norm;
+ StatValues_t small_block;
+ } curr;
StatValues_t max;
StatValues_t max_ever;
struct {
@@ -257,6 +276,8 @@ struct Allctr_t_ {
Uint largest_mbc_size;
Uint smallest_mbc_size;
Uint mbc_growth_stages;
+ Uint sbmbc_threshold;
+ Uint sbmbc_size;
#if HAVE_ERTS_MSEG
ErtsMsegOpt_t mseg_opt;
#endif
@@ -269,6 +290,7 @@ struct Allctr_t_ {
Uint min_block_size;
/* Carriers */
+ CarrierList_t sbmbc_list;
CarrierList_t mbc_list;
CarrierList_t sbc_list;
@@ -277,15 +299,15 @@ struct Allctr_t_ {
/* Callback functions (first 4 are mandatory) */
Block_t * (*get_free_block) (Allctr_t *, Uint,
- Block_t *, Uint);
- void (*link_free_block) (Allctr_t *, Block_t *);
- void (*unlink_free_block) (Allctr_t *, Block_t *);
+ Block_t *, Uint, Uint32);
+ void (*link_free_block) (Allctr_t *, Block_t *, Uint32);
+ void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32);
Eterm (*info_options) (Allctr_t *, char *, int *,
void *, Uint **, Uint *);
Uint (*get_next_mbc_size) (Allctr_t *);
- void (*creating_mbc) (Allctr_t *, Carrier_t *);
- void (*destroying_mbc) (Allctr_t *, Carrier_t *);
+ void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32);
+ void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32);
void (*init_atoms) (void);
#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
@@ -312,6 +334,8 @@ struct Allctr_t_ {
CallCounter_t this_alloc;
CallCounter_t this_free;
CallCounter_t this_realloc;
+ CallCounter_t sbmbc_alloc;
+ CallCounter_t sbmbc_free;
CallCounter_t mseg_alloc;
CallCounter_t mseg_dealloc;
CallCounter_t mseg_realloc;
@@ -322,6 +346,7 @@ struct Allctr_t_ {
CarriersStats_t sbcs;
CarriersStats_t mbcs;
+ CarriersStats_t sbmbcs;
#ifdef DEBUG
#ifdef USE_THREADS
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
new file mode 100644
index 0000000000..002852cdad
--- /dev/null
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -0,0 +1,972 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * Description: An "address order first fit" allocator
+ * based on a Red-Black (binary search) Tree. The search,
+ * insert, and delete operations are all O(log n) operations
+ * on a Red-Black Tree.
+ * Red-Black Trees are described in "Introduction to Algorithms",
+ * by Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Riverest.
+ *
+ * This module is a callback-module for erl_alloc_util.c
+ *
+ * Algorithm: The tree nodes are free-blocks ordered in address order.
+ * Every node also keeps the size of the largest block in its
+ * sub-tree ('max_size'). By that we can start from root and keep
+ * left (for low addresses) while dismissing entire sub-trees with
+ * too small blocks.
+ *
+ * Authors: Rickard Green/Sverker Eriksson
+ */
+
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+#include "global.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
+#ifdef DEBUG
+#if 0
+#define HARD_DEBUG
+#endif
+#else
+#undef HARD_DEBUG
+#endif
+
+#define MIN_MBC_SZ (16*1024)
+#define MIN_MBC_FIRST_FREE_SZ (4*1024)
+
+#define TREE_NODE_FLG (((Uint) 1) << 0)
+#define RED_FLG (((Uint) 1) << 1)
+#ifdef HARD_DEBUG
+# define LEFT_VISITED_FLG (((Uint) 1) << 2)
+# define RIGHT_VISITED_FLG (((Uint) 1) << 3)
+#endif
+
+#define IS_RED(N) (((AOFF_RBTree_t *) (N)) \
+ && ((AOFF_RBTree_t *) (N))->flags & RED_FLG)
+#define IS_BLACK(N) (!IS_RED(((AOFF_RBTree_t *) (N))))
+
+#define SET_RED(N) (((AOFF_RBTree_t *) (N))->flags |= RED_FLG)
+#define SET_BLACK(N) (((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG)
+
+#undef ASSERT
+#define ASSERT ASSERT_EXPR
+
+#if 1
+#define RBT_ASSERT ASSERT
+#else
+#define RBT_ASSERT(x)
+#endif
+
+
+/* Types... */
+typedef struct AOFF_RBTree_t_ AOFF_RBTree_t;
+
+struct AOFF_RBTree_t_ {
+ Block_t hdr;
+ Uint flags;
+ AOFF_RBTree_t *parent;
+ AOFF_RBTree_t *left;
+ AOFF_RBTree_t *right;
+ Uint max_sz; /* of all blocks in this sub-tree */
+};
+
+#ifdef HARD_DEBUG
+static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
+#endif
+
+
+/* Calculate 'max_size' of tree node x by only looking at the direct children
+ * of x and x itself.
+ */
+static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x)
+{
+ Uint sz = BLK_SZ(x);
+ if (x->left && x->left->max_sz > sz) {
+ sz = x->left->max_sz;
+ }
+ if (x->right && x->right->max_sz > sz) {
+ sz = x->right->max_sz;
+ }
+ return sz;
+}
+
+/* Set new possibly lower 'max_size' of node and propagate change toward root
+*/
+static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
+ AOFF_RBTree_t* stop_at)
+{
+ AOFF_RBTree_t* x = node;
+ Uint old_max = x->max_sz;
+ Uint new_max = node_max_size(x);
+
+ if (new_max < old_max) {
+ x->max_sz = new_max;
+ while ((x=x->parent) != stop_at && x->max_sz == old_max) {
+ x->max_sz = node_max_size(x);
+ }
+ ASSERT(x == stop_at || x->max_sz > old_max);
+ }
+ else ASSERT(new_max == old_max);
+}
+
+
+/* Prototypes of callback functions */
+static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags);
+static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags);
+static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags);
+
+static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *);
+static void init_atoms(void);
+
+
+
+#ifdef DEBUG
+
+/* Destroy all tree fields */
+#define DESTROY_TREE_NODE(N) \
+ sys_memset((void *) (((Block_t *) (N)) + 1), \
+ 0xff, \
+ (sizeof(AOFF_RBTree_t) - sizeof(Block_t)))
+
+#else
+
+#define DESTROY_TREE_NODE(N)
+
+#endif
+
+
+static int atoms_initialized = 0;
+
+void
+erts_aoffalc_init(void)
+{
+ atoms_initialized = 0;
+}
+
+Allctr_t *
+erts_aoffalc_start(AOFFAllctr_t *alc,
+ AOFFAllctrInit_t* aoffinit,
+ AllctrInit_t *init)
+{
+ AOFFAllctr_t nulled_state = {{0}};
+ /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc
+ warning. gcc warns if {0} is used as initializer of a struct when
+ the first member is a struct (not if, for example, the third member
+ is a struct). */
+ Allctr_t *allctr = (Allctr_t *) alc;
+
+ sys_memcpy((void *) alc, (void *) &nulled_state, sizeof(AOFFAllctr_t));
+
+ allctr->mbc_header_size = sizeof(Carrier_t);
+ allctr->min_mbc_size = MIN_MBC_SZ;
+ allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ;
+ allctr->min_block_size = sizeof(AOFF_RBTree_t);
+
+ allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR;
+
+
+ /* Callback functions */
+
+ allctr->get_free_block = aoff_get_free_block;
+ allctr->link_free_block = aoff_link_free_block;
+ allctr->unlink_free_block = aoff_unlink_free_block;
+ allctr->info_options = info_options;
+
+ allctr->get_next_mbc_size = NULL;
+ allctr->creating_mbc = NULL;
+ allctr->destroying_mbc = NULL;
+ allctr->init_atoms = init_atoms;
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+ allctr->check_block = NULL;
+ allctr->check_mbc = NULL;
+#endif
+
+ allctr->atoms_initialized = 0;
+
+ if (!erts_alcu_start(allctr, init))
+ return NULL;
+
+ return allctr;
+}
+
+/*
+ * Red-Black Tree operations needed
+ */
+
+static ERTS_INLINE void
+left_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x)
+{
+ AOFF_RBTree_t *y = x->right;
+ x->right = y->left;
+ if (y->left)
+ y->left->parent = x;
+ y->parent = x->parent;
+ if (!y->parent) {
+ RBT_ASSERT(*root == x);
+ *root = y;
+ }
+ else if (x == x->parent->left)
+ x->parent->left = y;
+ else {
+ RBT_ASSERT(x == x->parent->right);
+ x->parent->right = y;
+ }
+ y->left = x;
+ x->parent = y;
+
+ y->max_sz = x->max_sz;
+ x->max_sz = node_max_size(x);
+ ASSERT(y->max_sz >= x->max_sz);
+}
+
+static ERTS_INLINE void
+right_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x)
+{
+ AOFF_RBTree_t *y = x->left;
+ x->left = y->right;
+ if (y->right)
+ y->right->parent = x;
+ y->parent = x->parent;
+ if (!y->parent) {
+ RBT_ASSERT(*root == x);
+ *root = y;
+ }
+ else if (x == x->parent->right)
+ x->parent->right = y;
+ else {
+ RBT_ASSERT(x == x->parent->left);
+ x->parent->left = y;
+ }
+ y->right = x;
+ x->parent = y;
+ y->max_sz = x->max_sz;
+ x->max_sz = node_max_size(x);
+ ASSERT(y->max_sz >= x->max_sz);
+}
+
+
+/*
+ * Replace node x with node y
+ * NOTE: block header of y is not changed
+ */
+static ERTS_INLINE void
+replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y)
+{
+
+ if (!x->parent) {
+ RBT_ASSERT(*root == x);
+ *root = y;
+ }
+ else if (x == x->parent->left)
+ x->parent->left = y;
+ else {
+ RBT_ASSERT(x == x->parent->right);
+ x->parent->right = y;
+ }
+ if (x->left) {
+ RBT_ASSERT(x->left->parent == x);
+ x->left->parent = y;
+ }
+ if (x->right) {
+ RBT_ASSERT(x->right->parent == x);
+ x->right->parent = y;
+ }
+
+ y->flags = x->flags;
+ y->parent = x->parent;
+ y->right = x->right;
+ y->left = x->left;
+
+ y->max_sz = x->max_sz;
+ lower_max_size(y, NULL);
+ DESTROY_TREE_NODE(x);
+}
+
+static void
+tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk)
+{
+ AOFF_RBTree_t *x = blk, *y;
+
+ /*
+ * Rearrange the tree so that it satisfies the Red-Black Tree properties
+ */
+
+ RBT_ASSERT(x != *root && IS_RED(x->parent));
+ do {
+
+ /*
+ * x and its parent are both red. Move the red pair up the tree
+ * until we get to the root or until we can separate them.
+ */
+
+ RBT_ASSERT(IS_RED(x));
+ RBT_ASSERT(IS_BLACK(x->parent->parent));
+ RBT_ASSERT(x->parent->parent);
+
+ if (x->parent == x->parent->parent->left) {
+ y = x->parent->parent->right;
+ if (IS_RED(y)) {
+ SET_BLACK(y);
+ x = x->parent;
+ SET_BLACK(x);
+ x = x->parent;
+ SET_RED(x);
+ }
+ else {
+
+ if (x == x->parent->right) {
+ x = x->parent;
+ left_rotate(root, x);
+ }
+
+ RBT_ASSERT(x == x->parent->parent->left->left);
+ RBT_ASSERT(IS_RED(x));
+ RBT_ASSERT(IS_RED(x->parent));
+ RBT_ASSERT(IS_BLACK(x->parent->parent));
+ RBT_ASSERT(IS_BLACK(y));
+
+ SET_BLACK(x->parent);
+ SET_RED(x->parent->parent);
+ right_rotate(root, x->parent->parent);
+
+ RBT_ASSERT(x == x->parent->left);
+ RBT_ASSERT(IS_RED(x));
+ RBT_ASSERT(IS_RED(x->parent->right));
+ RBT_ASSERT(IS_BLACK(x->parent));
+ break;
+ }
+ }
+ else {
+ RBT_ASSERT(x->parent == x->parent->parent->right);
+ y = x->parent->parent->left;
+ if (IS_RED(y)) {
+ SET_BLACK(y);
+ x = x->parent;
+ SET_BLACK(x);
+ x = x->parent;
+ SET_RED(x);
+ }
+ else {
+
+ if (x == x->parent->left) {
+ x = x->parent;
+ right_rotate(root, x);
+ }
+
+ RBT_ASSERT(x == x->parent->parent->right->right);
+ RBT_ASSERT(IS_RED(x));
+ RBT_ASSERT(IS_RED(x->parent));
+ RBT_ASSERT(IS_BLACK(x->parent->parent));
+ RBT_ASSERT(IS_BLACK(y));
+
+ SET_BLACK(x->parent);
+ SET_RED(x->parent->parent);
+ left_rotate(root, x->parent->parent);
+
+ RBT_ASSERT(x == x->parent->right);
+ RBT_ASSERT(IS_RED(x));
+ RBT_ASSERT(IS_RED(x->parent->left));
+ RBT_ASSERT(IS_BLACK(x->parent));
+ break;
+ }
+ }
+ } while (x != *root && IS_RED(x->parent));
+
+ SET_BLACK(*root);
+}
+
+static void
+aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags)
+{
+ AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+ AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &alc->sbmbc_root : &alc->mbc_root);
+ Uint spliced_is_black;
+ AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del;
+ AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we
+ splice out a node without children. */
+
+ null_x.parent = NULL;
+
+#ifdef HARD_DEBUG
+ check_tree(*root, 0);
+#endif
+
+ /* Remove node from tree... */
+
+ /* Find node to splice out */
+ if (!z->left || !z->right)
+ y = z;
+ else
+ /* Set y to z:s successor */
+ for(y = z->right; y->left; y = y->left);
+ /* splice out y */
+ x = y->left ? y->left : y->right;
+ spliced_is_black = IS_BLACK(y);
+ if (x) {
+ x->parent = y->parent;
+ }
+ else if (spliced_is_black) {
+ x = &null_x;
+ x->flags = 0;
+ SET_BLACK(x);
+ x->right = x->left = NULL;
+ x->max_sz = 0;
+ x->parent = y->parent;
+ y->left = x;
+ }
+
+ if (!y->parent) {
+ RBT_ASSERT(*root == y);
+ *root = x;
+ }
+ else {
+ if (y == y->parent->left) {
+ y->parent->left = x;
+ }
+ else {
+ RBT_ASSERT(y == y->parent->right);
+ y->parent->right = x;
+ }
+ if (y->parent != z) {
+ lower_max_size(y->parent, (y==z ? NULL : z));
+ }
+ }
+ if (y != z) {
+ /* We spliced out the successor of z; replace z by the successor */
+ replace(root, z, y);
+ }
+
+ if (spliced_is_black) {
+ /* We removed a black node which makes the resulting tree
+ violate the Red-Black Tree properties. Fixup tree... */
+
+ while (IS_BLACK(x) && x->parent) {
+
+ /*
+ * x has an "extra black" which we move up the tree
+ * until we reach the root or until we can get rid of it.
+ *
+ * y is the sibbling of x
+ */
+
+ if (x == x->parent->left) {
+ y = x->parent->right;
+ RBT_ASSERT(y);
+ if (IS_RED(y)) {
+ RBT_ASSERT(y->right);
+ RBT_ASSERT(y->left);
+ SET_BLACK(y);
+ RBT_ASSERT(IS_BLACK(x->parent));
+ SET_RED(x->parent);
+ left_rotate(root, x->parent);
+ y = x->parent->right;
+ }
+ RBT_ASSERT(y);
+ RBT_ASSERT(IS_BLACK(y));
+ if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
+ SET_RED(y);
+ x = x->parent;
+ }
+ else {
+ if (IS_BLACK(y->right)) {
+ SET_BLACK(y->left);
+ SET_RED(y);
+ right_rotate(root, y);
+ y = x->parent->right;
+ }
+ RBT_ASSERT(y);
+ if (IS_RED(x->parent)) {
+
+ SET_BLACK(x->parent);
+ SET_RED(y);
+ }
+ RBT_ASSERT(y->right);
+ SET_BLACK(y->right);
+ left_rotate(root, x->parent);
+ x = *root;
+ break;
+ }
+ }
+ else {
+ RBT_ASSERT(x == x->parent->right);
+ y = x->parent->left;
+ RBT_ASSERT(y);
+ if (IS_RED(y)) {
+ RBT_ASSERT(y->right);
+ RBT_ASSERT(y->left);
+ SET_BLACK(y);
+ RBT_ASSERT(IS_BLACK(x->parent));
+ SET_RED(x->parent);
+ right_rotate(root, x->parent);
+ y = x->parent->left;
+ }
+ RBT_ASSERT(y);
+ RBT_ASSERT(IS_BLACK(y));
+ if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
+ SET_RED(y);
+ x = x->parent;
+ }
+ else {
+ if (IS_BLACK(y->left)) {
+ SET_BLACK(y->right);
+ SET_RED(y);
+ left_rotate(root, y);
+ y = x->parent->left;
+ }
+ RBT_ASSERT(y);
+ if (IS_RED(x->parent)) {
+ SET_BLACK(x->parent);
+ SET_RED(y);
+ }
+ RBT_ASSERT(y->left);
+ SET_BLACK(y->left);
+ right_rotate(root, x->parent);
+ x = *root;
+ break;
+ }
+ }
+ }
+ SET_BLACK(x);
+
+ if (null_x.parent) {
+ if (null_x.parent->left == &null_x)
+ null_x.parent->left = NULL;
+ else {
+ RBT_ASSERT(null_x.parent->right == &null_x);
+ null_x.parent->right = NULL;
+ }
+ RBT_ASSERT(!null_x.left);
+ RBT_ASSERT(!null_x.right);
+ }
+ else if (*root == &null_x) {
+ *root = NULL;
+ RBT_ASSERT(!null_x.left);
+ RBT_ASSERT(!null_x.right);
+ }
+ }
+
+ DESTROY_TREE_NODE(del);
+
+#ifdef HARD_DEBUG
+ check_tree(*root, 0);
+#endif
+}
+
+static void
+aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
+{
+ AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+ AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block;
+ AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &alc->sbmbc_root : &alc->mbc_root);
+ Uint blk_sz = BLK_SZ(blk);
+
+#ifdef HARD_DEBUG
+ check_tree(*root, 0);
+#endif
+
+ blk->flags = 0;
+ blk->left = NULL;
+ blk->right = NULL;
+ blk->max_sz = blk_sz;
+
+ if (!*root) {
+ blk->parent = NULL;
+ SET_BLACK(blk);
+ *root = blk;
+ }
+ else {
+ AOFF_RBTree_t *x = *root;
+ while (1) {
+ if (x->max_sz < blk_sz) {
+ x->max_sz = blk_sz;
+ }
+ if (blk < x) {
+ if (!x->left) {
+ blk->parent = x;
+ x->left = blk;
+ break;
+ }
+ x = x->left;
+ }
+ else {
+ if (!x->right) {
+ blk->parent = x;
+ x->right = blk;
+ break;
+ }
+ x = x->right;
+ }
+
+ }
+
+ /* Insert block into size tree */
+ RBT_ASSERT(blk->parent);
+
+ SET_RED(blk);
+ if (IS_RED(blk->parent))
+ tree_insert_fixup(root, blk);
+ }
+
+#ifdef HARD_DEBUG
+ check_tree(*root, 0);
+#endif
+}
+
+static Block_t *
+aoff_get_free_block(Allctr_t *allctr, Uint size,
+ Block_t *cand_blk, Uint cand_size, Uint32 flags)
+{
+ AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+ AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? alc->sbmbc_root : alc->mbc_root);
+ AOFF_RBTree_t *blk = NULL;
+#ifdef HARD_DEBUG
+ AOFF_RBTree_t* dbg_blk = check_tree(x, size);
+#endif
+
+ ASSERT(!cand_blk || cand_size >= size);
+
+ while (x) {
+ if (x->left && x->left->max_sz >= size) {
+ x = x->left;
+ }
+ else if (BLK_SZ(x) >= size) {
+ blk = x;
+ break;
+ }
+ else {
+ x = x->right;
+ }
+ }
+
+#ifdef HARD_DEBUG
+ ASSERT(blk == dbg_blk);
+#endif
+
+ if (!blk)
+ return NULL;
+
+ if (cand_blk && cand_blk < &blk->hdr) {
+ return NULL; /* cand_blk was better */
+ }
+
+ aoff_unlink_free_block(allctr, (Block_t *) blk, flags);
+
+ return (Block_t *) blk;
+}
+
+
+/*
+ * info_options()
+ */
+
+static struct {
+ Eterm as;
+ Eterm aoff;
+#ifdef DEBUG
+ Eterm end_of_atoms;
+#endif
+} am;
+
+static void ERTS_INLINE atom_init(Eterm *atom, char *name)
+{
+ *atom = am_atom_put(name, strlen(name));
+}
+#define AM_INIT(AM) atom_init(&am.AM, #AM)
+
+static void
+init_atoms(void)
+{
+#ifdef DEBUG
+ Eterm *atom;
+#endif
+
+ if (atoms_initialized)
+ return;
+
+#ifdef DEBUG
+ for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
+ *atom = THE_NON_VALUE;
+ }
+#endif
+ AM_INIT(as);
+ AM_INIT(aoff);
+
+#ifdef DEBUG
+ for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
+ ASSERT(*atom != THE_NON_VALUE);
+ }
+#endif
+
+ atoms_initialized = 1;
+}
+
+
+#define bld_uint erts_bld_uint
+#define bld_cons erts_bld_cons
+#define bld_tuple erts_bld_tuple
+
+static ERTS_INLINE void
+add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
+{
+ *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp);
+}
+
+static Eterm
+info_options(Allctr_t *allctr,
+ char *prefix,
+ int *print_to_p,
+ void *print_to_arg,
+ Uint **hpp,
+ Uint *szp)
+{
+ Eterm res = THE_NON_VALUE;
+
+ if (print_to_p) {
+ erts_print(*print_to_p,
+ print_to_arg,
+ "%sas: %s\n",
+ prefix,
+ "aoff");
+ }
+
+ if (hpp || szp) {
+
+ if (!atoms_initialized)
+ erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+ __FILE__, __LINE__);;
+
+ res = NIL;
+ add_2tup(hpp, szp, &res, am.as, am.aoff);
+ }
+
+ return res;
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * NOTE: erts_aoffalc_test() is only supposed to be used for testing. *
+ * *
+ * Keep alloc_SUITE_data/allocator_test.h updated if changes are made *
+ * to erts_aoffalc_test() *
+\* */
+
+unsigned long
+erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2)
+{
+ switch (op) {
+ case 0x500: return (unsigned long) 0; /* IS_AOBF */
+ case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root;
+ case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent;
+ case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left;
+ case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right;
+ case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1);
+ case 0x508: return (unsigned long) 1; /* IS_AOFF */
+ case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz;
+ default: ASSERT(0); return ~((unsigned long) 0);
+ }
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Debug functions *
+\* */
+
+
+#ifdef HARD_DEBUG
+
+#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG)
+#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG)
+
+#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG)
+#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG)
+
+#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG)
+#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG)
+
+
+#if 0
+# define PRINT_TREE
+#else
+# undef PRINT_TREE
+#endif
+
+#ifdef PRINT_TREE
+static void print_tree(AOFF_RBTree_t*);
+#endif
+
+/*
+ * Checks that the order between parent and children are correct,
+ * and that the Red-Black Tree properies are satisfied. if size > 0,
+ * check_tree() returns the node that satisfies "address order first fit"
+ *
+ * The Red-Black Tree properies are:
+ * 1. Every node is either red or black.
+ * 2. Every leaf (NIL) is black.
+ * 3. If a node is red, then both its children are black.
+ * 4. Every simple path from a node to a descendant leaf
+ * contains the same number of black nodes.
+ *
+ * + own.max_size == MAX(own.size, left.max_size, right.max_size)
+ */
+
+static AOFF_RBTree_t *
+check_tree(AOFF_RBTree_t* root, Uint size)
+{
+ AOFF_RBTree_t *res = NULL;
+ Sint blacks;
+ Sint curr_blacks;
+ AOFF_RBTree_t *x;
+
+#ifdef PRINT_TREE
+ print_tree(root);
+#endif
+
+ if (!root)
+ return res;
+
+ x = root;
+ ASSERT(IS_BLACK(x));
+ ASSERT(!x->parent);
+ curr_blacks = 1;
+ blacks = -1;
+
+ while (x) {
+ if (!IS_LEFT_VISITED(x)) {
+ SET_LEFT_VISITED(x);
+ if (x->left) {
+ x = x->left;
+ if (IS_BLACK(x))
+ curr_blacks++;
+ continue;
+ }
+ else {
+ if (blacks < 0)
+ blacks = curr_blacks;
+ ASSERT(blacks == curr_blacks);
+ }
+ }
+
+ if (!IS_RIGHT_VISITED(x)) {
+ SET_RIGHT_VISITED(x);
+ if (x->right) {
+ x = x->right;
+ if (IS_BLACK(x))
+ curr_blacks++;
+ continue;
+ }
+ else {
+ if (blacks < 0)
+ blacks = curr_blacks;
+ ASSERT(blacks == curr_blacks);
+ }
+ }
+
+
+ if (IS_RED(x)) {
+ ASSERT(IS_BLACK(x->right));
+ ASSERT(IS_BLACK(x->left));
+ }
+
+ ASSERT(x->parent || x == root);
+
+ if (x->left) {
+ ASSERT(x->left->parent == x);
+ ASSERT(x->left < x);
+ ASSERT(x->left->max_sz <= x->max_sz);
+ }
+
+ if (x->right) {
+ ASSERT(x->right->parent == x);
+ ASSERT(x->right > x);
+ ASSERT(x->right->max_sz <= x->max_sz);
+ }
+ ASSERT(x->max_sz >= BLK_SZ(x));
+ ASSERT(x->max_sz == BLK_SZ(x)
+ || x->max_sz == (x->left ? x->left->max_sz : 0)
+ || x->max_sz == (x->right ? x->right->max_sz : 0));
+
+ if (size && BLK_SZ(x) >= size) {
+ if (!res || x < res) {
+ res = x;
+ }
+ }
+
+ UNSET_LEFT_VISITED(x);
+ UNSET_RIGHT_VISITED(x);
+ if (IS_BLACK(x))
+ curr_blacks--;
+ x = x->parent;
+
+ }
+
+ ASSERT(curr_blacks == 0);
+
+ UNSET_LEFT_VISITED(root);
+ UNSET_RIGHT_VISITED(root);
+
+ return res;
+
+}
+
+
+#ifdef PRINT_TREE
+#define INDENT_STEP 2
+
+#include <stdio.h>
+
+static void
+print_tree_aux(AOFF_RBTree_t *x, int indent)
+{
+ int i;
+
+ if (x) {
+ print_tree_aux(x->right, indent + INDENT_STEP);
+ for (i = 0; i < indent; i++) {
+ putc(' ', stderr);
+ }
+ fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n",
+ IS_BLACK(x) ? "BLACK" : "RED",
+ BLK_SZ(x), (Uint)x, x->max_sz);
+ print_tree_aux(x->left, indent + INDENT_STEP);
+ }
+}
+
+
+static void
+print_tree(AOFF_RBTree_t* root)
+{
+ fprintf(stderr, " --- AOFF tree begin ---\r\n");
+ print_tree_aux(root, 0);
+ fprintf(stderr, " --- AOFF tree end ---\r\n");
+}
+
+#endif /* PRINT_TREE */
+
+#endif /* HARD_DEBUG */
+
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
new file mode 100644
index 0000000000..0bf0ec8cee
--- /dev/null
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -0,0 +1,60 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+#ifndef ERL_AO_FIRSTFIT_ALLOC__
+#define ERL_AO_FIRSTFIT_ALLOC__
+
+#include "erl_alloc_util.h"
+
+#define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9"
+
+typedef struct AOFFAllctr_t_ AOFFAllctr_t;
+
+typedef struct {
+ int dummy;
+} AOFFAllctrInit_t;
+
+#define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/}
+
+void erts_aoffalc_init(void);
+Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *);
+
+#endif /* #ifndef ERL_AO_FIRSTFIT_ALLOC__ */
+
+
+
+#if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__)
+#define ERL_AOFF_ALLOC_IMPL__
+
+#define GET_ERL_ALLOC_UTIL_IMPL
+#include "erl_alloc_util.h"
+
+
+struct AOFFAllctr_t_ {
+ Allctr_t allctr; /* Has to be first! */
+
+ struct AOFF_RBTree_t_* mbc_root;
+ struct AOFF_RBTree_t_* sbmbc_root;
+};
+
+unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long);
+
+#endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL)
+ && !defined(ERL_AOFF_ALLOC_IMPL__) */
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 3035e5df16..5e3032ddaa 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -84,24 +84,24 @@
#ifdef HARD_DEBUG
-static RBTree_t * check_tree(BFAllctr_t *, Uint);
+static RBTree_t * check_tree(RBTree_t, int, Uint);
#endif
-static void tree_delete(Allctr_t *allctr, Block_t *del);
+static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags);
/* Prototypes of callback functions */
/* "address order best fit" specific callback functions */
static Block_t * aobf_get_free_block (Allctr_t *, Uint,
- Block_t *, Uint);
-static void aobf_link_free_block (Allctr_t *, Block_t *);
+ Block_t *, Uint, Uint32);
+static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32);
#define aobf_unlink_free_block tree_delete
/* "best fit" specific callback functions */
static Block_t * bf_get_free_block (Allctr_t *, Uint,
- Block_t *, Uint);
-static void bf_link_free_block (Allctr_t *, Block_t *);
-static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *);
+ Block_t *, Uint, Uint32);
+static void bf_link_free_block (Allctr_t *, Block_t *, Uint32);
+static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32);
static Eterm info_options (Allctr_t *, char *, int *,
@@ -303,7 +303,7 @@ replace(RBTree_t **root, RBTree_t *x, RBTree_t *y)
}
static void
-tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
+tree_insert_fixup(RBTree_t **root, RBTree_t *blk)
{
RBTree_t *x = blk, *y;
@@ -311,7 +311,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
* Rearrange the tree so that it satisfies the Red-Black Tree properties
*/
- RBT_ASSERT(x != bfallctr->root && IS_RED(x->parent));
+ RBT_ASSERT(x != *root && IS_RED(x->parent));
do {
/*
@@ -336,7 +336,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
if (x == x->parent->right) {
x = x->parent;
- left_rotate(&bfallctr->root, x);
+ left_rotate(root, x);
}
RBT_ASSERT(x == x->parent->parent->left->left);
@@ -347,7 +347,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
SET_BLACK(x->parent);
SET_RED(x->parent->parent);
- right_rotate(&bfallctr->root, x->parent->parent);
+ right_rotate(root, x->parent->parent);
RBT_ASSERT(x == x->parent->left);
RBT_ASSERT(IS_RED(x));
@@ -370,7 +370,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
if (x == x->parent->left) {
x = x->parent;
- right_rotate(&bfallctr->root, x);
+ right_rotate(root, x);
}
RBT_ASSERT(x == x->parent->parent->right->right);
@@ -381,7 +381,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
SET_BLACK(x->parent);
SET_RED(x->parent->parent);
- left_rotate(&bfallctr->root, x->parent->parent);
+ left_rotate(root, x->parent->parent);
RBT_ASSERT(x == x->parent->right);
RBT_ASSERT(IS_RED(x));
@@ -390,9 +390,9 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
break;
}
}
- } while (x != bfallctr->root && IS_RED(x->parent));
+ } while (x != *root && IS_RED(x->parent));
- SET_BLACK(bfallctr->root);
+ SET_BLACK(*root);
}
@@ -402,18 +402,22 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
* callback function in the address order case.
*/
static void
-tree_delete(Allctr_t *allctr, Block_t *del)
+tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
Uint spliced_is_black;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
RBTree_t *x, *y, *z = (RBTree_t *) del;
RBTree_t null_x; /* null_x is used to get the fixup started when we
splice out a node without children. */
null_x.parent = NULL;
+
#ifdef HARD_DEBUG
- check_tree(bfallctr, 0);
+ check_tree(*root, bfallctr->address_order, 0);
#endif
/* Remove node from tree... */
@@ -440,8 +444,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
}
if (!y->parent) {
- RBT_ASSERT(bfallctr->root == y);
- bfallctr->root = x;
+ RBT_ASSERT(*root == y);
+ *root = x;
}
else if (y == y->parent->left)
y->parent->left = x;
@@ -451,7 +455,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
}
if (y != z) {
/* We spliced out the successor of z; replace z by the successor */
- replace(&bfallctr->root, z, y);
+ replace(root, z, y);
}
if (spliced_is_black) {
@@ -476,7 +480,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
- left_rotate(&bfallctr->root, x->parent);
+ left_rotate(root, x->parent);
y = x->parent->right;
}
RBT_ASSERT(y);
@@ -489,7 +493,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
if (IS_BLACK(y->right)) {
SET_BLACK(y->left);
SET_RED(y);
- right_rotate(&bfallctr->root, y);
+ right_rotate(root, y);
y = x->parent->right;
}
RBT_ASSERT(y);
@@ -500,8 +504,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
}
RBT_ASSERT(y->right);
SET_BLACK(y->right);
- left_rotate(&bfallctr->root, x->parent);
- x = bfallctr->root;
+ left_rotate(root, x->parent);
+ x = *root;
break;
}
}
@@ -515,7 +519,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
- right_rotate(&bfallctr->root, x->parent);
+ right_rotate(root, x->parent);
y = x->parent->left;
}
RBT_ASSERT(y);
@@ -528,7 +532,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
if (IS_BLACK(y->left)) {
SET_BLACK(y->right);
SET_RED(y);
- left_rotate(&bfallctr->root, y);
+ left_rotate(root, y);
y = x->parent->left;
}
RBT_ASSERT(y);
@@ -538,8 +542,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
}
RBT_ASSERT(y->left);
SET_BLACK(y->left);
- right_rotate(&bfallctr->root, x->parent);
- x = bfallctr->root;
+ right_rotate(root, x->parent);
+ x = *root;
break;
}
}
@@ -556,8 +560,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
- else if (bfallctr->root == &null_x) {
- bfallctr->root = NULL;
+ else if (*root == &null_x) {
+ *root = NULL;
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
@@ -567,7 +571,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
DESTROY_TREE_NODE(del);
#ifdef HARD_DEBUG
- check_tree(bfallctr, 0);
+ check_tree(root, bfallctr->address_order, 0);
#endif
}
@@ -577,23 +581,28 @@ tree_delete(Allctr_t *allctr, Block_t *del)
\* */
static void
-aobf_link_free_block(Allctr_t *allctr, Block_t *block)
+aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
RBTree_t *blk = (RBTree_t *) block;
Uint blk_sz = BLK_SZ(blk);
+
+
blk->flags = 0;
blk->left = NULL;
blk->right = NULL;
- if (!bfallctr->root) {
+ if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
- bfallctr->root = blk;
+ *root = blk;
}
else {
- RBTree_t *x = bfallctr->root;
+ RBTree_t *x = *root;
while (1) {
Uint size;
@@ -623,28 +632,32 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block)
SET_RED(blk);
if (IS_RED(blk->parent))
- tree_insert_fixup(bfallctr, blk);
+ tree_insert_fixup(root, blk);
}
#ifdef HARD_DEBUG
- check_tree(bfallctr, 0);
+ check_tree(root, 1, 0);
#endif
}
#if 0 /* tree_delete() is directly used instead */
static void
-aobf_unlink_free_block(Allctr_t *allctr, Block_t *block)
+aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
- tree_delete(allctr, block);
+ tree_delete(allctr, block, flags);
}
#endif
static Block_t *
aobf_get_free_block(Allctr_t *allctr, Uint size,
- Block_t *cand_blk, Uint cand_size)
+ Block_t *cand_blk, Uint cand_size,
+ Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
- RBTree_t *x = bfallctr->root;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
+ RBTree_t *x = *root;
RBTree_t *blk = NULL;
Uint blk_sz;
@@ -665,7 +678,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
return NULL;
#ifdef HARD_DEBUG
- ASSERT(blk == check_tree(bfallctr, size));
+ ASSERT(blk == check_tree(root, 1, size));
#endif
if (cand_blk) {
@@ -676,7 +689,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
return NULL; /* cand_blk was better */
}
- aobf_unlink_free_block(allctr, (Block_t *) blk);
+ aobf_unlink_free_block(allctr, (Block_t *) blk, flags);
return (Block_t *) blk;
}
@@ -687,9 +700,12 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
\* */
static void
-bf_link_free_block(Allctr_t *allctr, Block_t *block)
+bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
RBTree_t *blk = (RBTree_t *) block;
Uint blk_sz = BLK_SZ(blk);
@@ -700,13 +716,13 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
blk->left = NULL;
blk->right = NULL;
- if (!bfallctr->root) {
+ if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
- bfallctr->root = blk;
+ *root = blk;
}
else {
- RBTree_t *x = bfallctr->root;
+ RBTree_t *x = *root;
while (1) {
Uint size;
@@ -745,7 +761,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
SET_RED(blk);
if (IS_RED(blk->parent))
- tree_insert_fixup(bfallctr, blk);
+ tree_insert_fixup(root, blk);
}
@@ -753,14 +769,17 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
LIST_NEXT(blk) = NULL;
#ifdef HARD_DEBUG
- check_tree(bfallctr, 0);
+ check_tree(root, 0, 0);
#endif
}
static ERTS_INLINE void
-bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
+bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
RBTree_t *x = (RBTree_t *) block;
if (IS_LIST_ELEM(x)) {
@@ -778,9 +797,9 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
ASSERT(IS_LIST_ELEM(LIST_NEXT(x)));
#ifdef HARD_DEBUG
- check_tree(bfallctr, 0);
+ check_tree(root, 0, 0);
#endif
- replace(&bfallctr->root, x, LIST_NEXT(x));
+ replace(root, x, LIST_NEXT(x));
#ifdef HARD_DEBUG
check_tree(bfallctr, 0);
@@ -788,7 +807,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
}
else {
/* Remove from tree */
- tree_delete(allctr, block);
+ tree_delete(allctr, block, flags);
}
DESTROY_LIST_ELEM(x);
@@ -797,10 +816,14 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
static Block_t *
bf_get_free_block(Allctr_t *allctr, Uint size,
- Block_t *cand_blk, Uint cand_size)
+ Block_t *cand_blk, Uint cand_size,
+ Uint32 flags)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
- RBTree_t *x = bfallctr->root;
+ RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+ ? &bfallctr->sbmbc_root
+ : &bfallctr->mbc_root);
+ RBTree_t *x = *root;
RBTree_t *blk = NULL;
Uint blk_sz;
@@ -827,7 +850,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
#ifdef HARD_DEBUG
{
- RBTree_t *ct_blk = check_tree(bfallctr, size);
+ RBTree_t *ct_blk = check_tree(root, 0, size);
ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk));
}
#endif
@@ -839,7 +862,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
the tree node */
blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk;
- bf_unlink_free_block(allctr, (Block_t *) blk);
+ bf_unlink_free_block(allctr, (Block_t *) blk, flags);
return (Block_t *) blk;
}
@@ -949,13 +972,14 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2)
{
switch (op) {
case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order;
- case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->root;
+ case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root;
case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent;
case 0x203: return (unsigned long) ((RBTree_t *) a1)->left;
case 0x204: return (unsigned long) ((RBTree_t *) a1)->right;
case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next;
case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1);
case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1);
+ case 0x208: return (unsigned long) 0; /* IS_AOFF */
default: ASSERT(0); return ~((unsigned long) 0);
}
}
@@ -985,7 +1009,7 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2)
#endif
#ifdef PRINT_TREE
-static void print_tree(BFAllctr_t *);
+static void print_tree(RBTree_t *, int);
#endif
/*
@@ -1003,7 +1027,7 @@ static void print_tree(BFAllctr_t *);
*/
static RBTree_t *
-check_tree(BFAllctr_t *bfallctr, Uint size)
+check_tree(RBTree_t *root, int ao, Uint size)
{
RBTree_t *res = NULL;
Sint blacks;
@@ -1011,13 +1035,13 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
RBTree_t *x;
#ifdef PRINT_TREE
- print_tree(bfallctr);
+ print_tree(root, ao);
#endif
- if (!bfallctr->root)
+ if (!root)
return res;
- x = bfallctr->root;
+ x = root;
ASSERT(IS_BLACK(x));
ASSERT(!x->parent);
curr_blacks = 1;
@@ -1060,11 +1084,11 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
ASSERT(IS_BLACK(x->left));
}
- ASSERT(x->parent || x == bfallctr->root);
+ ASSERT(x->parent || x == root);
if (x->left) {
ASSERT(x->left->parent == x);
- if (bfallctr->address_order) {
+ if (ao) {
ASSERT(BLK_SZ(x->left) < BLK_SZ(x)
|| (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x));
}
@@ -1076,7 +1100,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
if (x->right) {
ASSERT(x->right->parent == x);
- if (bfallctr->address_order) {
+ if (ao) {
ASSERT(BLK_SZ(x->right) > BLK_SZ(x)
|| (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x));
}
@@ -1087,7 +1111,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
}
if (size && BLK_SZ(x) >= size) {
- if (bfallctr->address_order) {
+ if (ao) {
if (!res
|| BLK_SZ(x) < BLK_SZ(res)
|| (BLK_SZ(x) == BLK_SZ(res) && x < res))
@@ -1109,8 +1133,8 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
ASSERT(curr_blacks == 0);
- UNSET_LEFT_VISITED(bfallctr->root);
- UNSET_RIGHT_VISITED(bfallctr->root);
+ UNSET_LEFT_VISITED(root);
+ UNSET_RIGHT_VISITED(root);
return res;
@@ -1148,11 +1172,11 @@ print_tree_aux(RBTree_t *x, int indent)
static void
-print_tree(BFAllctr_t *bfallctr)
+print_tree(RBTree_t *root, int ao)
{
- char *type = bfallctr->address_order ? "Size-Adress" : "Size";
+ char *type = ao ? "Size-Adress" : "Size";
fprintf(stderr, " --- %s tree begin ---\r\n", type);
- print_tree_aux(bfallctr->root, 0);
+ print_tree_aux(root, 0);
fprintf(stderr, " --- %s tree end ---\r\n", type);
}
diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h
index cb35e21e57..faa2d9742e 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.h
+++ b/erts/emulator/beam/erl_bestfit_alloc.h
@@ -54,7 +54,8 @@ typedef struct RBTree_t_ RBTree_t;
struct BFAllctr_t_ {
Allctr_t allctr; /* Has to be first! */
- RBTree_t * root;
+ RBTree_t * mbc_root;
+ RBTree_t * sbmbc_root;
int address_order;
};
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 9631fb50db..d714eacd06 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -369,7 +369,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
if (!(prt->status & FREE_PORT_FLAGS) &&
prt->drv_ptr->handle == dh) {
#if DDLL_SMP
- erts_smp_atomic_inc(&prt->refc);
+ erts_smp_atomic_inc_nob(&prt->refc);
/* Extremely rare spinlock */
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
@@ -597,7 +597,7 @@ done:
if (!(prt->status & FREE_PORT_FLAGS)
&& prt->drv_ptr->handle == dh) {
#if DDLL_SMP
- erts_smp_atomic_inc(&prt->refc);
+ erts_smp_atomic_inc_nob(&prt->refc);
/* Extremely rare spinlock */
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
@@ -1054,7 +1054,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
if (!(prt->status & FREE_PORT_FLAGS) &&
prt->drv_ptr->handle == dh) {
#if DDLL_SMP
- erts_smp_atomic_inc(&prt->refc);
+ erts_smp_atomic_inc_nob(&prt->refc);
while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
erts_smp_port_state_unlock(prt);
erts_smp_port_state_lock(prt);
@@ -1602,7 +1602,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
erts_sys_ddll_close(dh->handle);
return ERL_DE_LOAD_ERROR_BAD_NAME;
}
- erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0);
+ erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
dh->port_count = 0;
dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
sys_strcpy(dh->full_path, path);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f264bf44df..6a74596f76 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -135,7 +135,7 @@ bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
if (szp)
*szp += 4+2;
if (hpp) {
- Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc);
+ Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc);
tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
res = CONS(*hpp + 4, tuple, res);
*hpp += 4+2;
@@ -2026,7 +2026,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
res = TUPLE2(hp, am_sequential_tracer, val);
BIF_RET(res);
} else if (BIF_ARG_1 == am_garbage_collection){
- Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs);
+ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
Eterm tup;
hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2);
@@ -2041,7 +2041,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
BIF_RET(res);
} else if (BIF_ARG_1 == am_fullsweep_after){
- Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs);
+ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
hp = HAlloc(BIF_P, 3);
res = TUPLE2(hp, am_fullsweep_after, make_small(val));
BIF_RET(res);
@@ -2545,6 +2545,70 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
BIF_RET(res);
+ } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) {
+ int i;
+ char **str;
+#ifdef ETHR_NATIVE_ATOMIC32_IMPL
+ erts_printf("32-bit native atomics: %s\n",
+ ETHR_NATIVE_ATOMIC32_IMPL);
+ str = ethr_native_atomic32_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_atomic32_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_ATOMIC64_IMPL
+ erts_printf("64-bit native atomics: %s\n",
+ ETHR_NATIVE_ATOMIC64_IMPL);
+ str = ethr_native_atomic64_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_atomic64_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
+ if (ethr_have_native_dw_atomic()) {
+ erts_printf("Double word native atomics: %s\n",
+ ETHR_NATIVE_DW_ATOMIC_IMPL);
+ str = ethr_native_dw_atomic_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_dw_atomic_%s()\n", str[i]);
+ str = ethr_native_su_dw_atomic_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]);
+ }
+#endif
+#ifdef ETHR_NATIVE_SPINLOCK_IMPL
+ erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL);
+#endif
+#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
+ erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL);
+#endif
+#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ ? "yes" : "no"));
+#endif
+#ifdef ETHR_X86_OUT_OF_ORDER
+ erts_printf("x86"
+#ifdef ARCH_64
+ "_64"
+#endif
+ " out of order\n");
+#endif
+#ifdef ETHR_SPARC_TSO
+ erts_printf("Sparc TSO\n");
+#endif
+#ifdef ETHR_SPARC_PSO
+ erts_printf("Sparc PSO\n");
+#endif
+#ifdef ETHR_SPARC_RMO
+ erts_printf("Sparc RMO\n");
+#endif
+#if defined(ETHR_PPC_HAVE_LWSYNC)
+ erts_printf("Have lwsync instruction: yes\n");
+#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
+ erts_printf("Have lwsync instruction: no\n");
+#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
+ erts_printf("Have lwsync instruction: %s (runtime test)\n",
+ ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no");
+#endif
+ BIF_RET(am_true);
}
BIF_ERROR(BIF_P, BADARG);
@@ -2845,7 +2909,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what)
}
break;
case am_refc:
- val = erts_make_integer(erts_smp_atomic_read(&funp->fe->refc), p);
+ val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p);
hp = HAlloc(p, 3);
break;
case am_arity:
@@ -3065,8 +3129,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
Eterm r1, r2;
Eterm in, out;
Uint hsz = 9;
- Uint bytes_in = (Uint) erts_smp_atomic_read(&erts_bytes_in);
- Uint bytes_out = (Uint) erts_smp_atomic_read(&erts_bytes_out);
+ Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in);
+ Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out);
(void) erts_bld_uint(NULL, &hsz, bytes_in);
(void) erts_bld_uint(NULL, &hsz, bytes_out);
@@ -3139,7 +3203,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
* NOTE: Only supposed to be used for testing, and debugging.
*/
- if (!erts_smp_atomic_read(&available_internal_state)) {
+ if (!erts_smp_atomic_read_nob(&available_internal_state)) {
BIF_ERROR(BIF_P, EXC_UNDEF);
}
@@ -3437,7 +3501,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)
&& (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) {
erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true);
- erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on);
+ erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on);
if (on) {
erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
erts_dsprintf(dsbufp, "Process %T ", BIF_P->id);
@@ -3453,7 +3517,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
BIF_RET(prev_on ? am_true : am_false);
}
- if (!erts_smp_atomic_read(&available_internal_state)) {
+ if (!erts_smp_atomic_read_nob(&available_internal_state)) {
BIF_ERROR(BIF_P, EXC_UNDEF);
}
@@ -3634,14 +3698,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
}
else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) {
/* Used by hipe test suites */
- erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag);
+ erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag);
if (!flag && BIF_ARG_2 != am_false) {
- erts_smp_atomic_set(&hipe_test_reschedule_flag, 1);
+ erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1);
erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2],
BIF_P, BIF_ARG_1, BIF_ARG_2);
}
- erts_smp_atomic_set(&hipe_test_reschedule_flag, !flag);
+ erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag);
BIF_RET(NIL);
}
else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) {
@@ -3951,8 +4015,8 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
void
erts_bif_info_init(void)
{
- erts_smp_atomic_init(&available_internal_state, 0);
- erts_smp_atomic_init(&hipe_test_reschedule_flag, 0);
+ erts_smp_atomic_init_nob(&available_internal_state, 0);
+ erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0);
process_info_init();
}
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index e56084b9cb..326a5c136b 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -76,14 +76,12 @@ struct erl_bits_state ErlBitsState;
#define byte_buf (ErlBitsState.byte_buf_)
#define byte_buf_len (ErlBitsState.byte_buf_len_)
-#ifdef ERTS_SMP
static erts_smp_atomic_t bits_bufs_size;
-#endif
Uint
erts_bits_bufs_size(void)
{
- return 0;
+ return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size);
}
#if !defined(ERTS_SMP)
@@ -109,8 +107,8 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0)
void
erts_init_bits(void)
{
+ erts_smp_atomic_init_nob(&bits_bufs_size, 0);
#if defined(ERTS_SMP)
- erts_smp_atomic_init(&bits_bufs_size, 0);
/* erl_process.c calls erts_bits_init_state() on all state instances */
#else
ERL_BITS_DECLARE_STATEP;
@@ -713,9 +711,7 @@ static void
ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need))
{
if (byte_buf_len < need) {
-#ifdef ERTS_SMP
- erts_smp_atomic_add(&bits_bufs_size, need - byte_buf_len);
-#endif
+ erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len);
byte_buf_len = need;
byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len);
}
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
index bcf8bcf270..03c0ef904a 100644
--- a/erts/emulator/beam/erl_cpu_topology.c
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -487,7 +487,7 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
/* Make sure we check if we should bind to a cpu or not... */
if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
- erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1);
+ erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 1);
else
esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
}
@@ -503,7 +503,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
erts_cpu_groups_callback_call_t *cgcc;
#ifdef ERTS_SMP
if (erts_common_run_queue)
- erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0);
+ erts_smp_atomic32_set_nob(&esdp->chk_cpu_bind, 0);
else {
esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
}
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index e0a6aa05c6..59ef8fc3ea 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -224,21 +224,21 @@ static void
free_dbtable(DbTable* tb)
{
#ifdef HARDDEBUG
- if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) {
+ if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) {
erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
- erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable),
+ erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable),
tb->common.fixations);
}
erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
tb->common.id);
erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
- erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
+ erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
- erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
+ erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
#endif
#ifdef ERTS_SMP
@@ -248,6 +248,7 @@ free_dbtable(DbTable* tb)
ASSERT(is_immed(tb->common.heir_data));
erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
+ ERTS_THR_MEMORY_BARRIER;
}
#ifdef ERTS_SMP
@@ -338,13 +339,13 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
if (tb->common.type & DB_FINE_LOCKED) {
- if (tb->common.is_thread_safe) {
- ASSERT(kind == LCK_WRITE);
+ if (kind == LCK_WRITE) {
+ ASSERT(tb->common.is_thread_safe);
tb->common.is_thread_safe = 0;
erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
}
else {
- ASSERT(kind != LCK_WRITE);
+ ASSERT(!tb->common.is_thread_safe);
erts_smp_rwmtx_runlock(&tb->common.rwlock);
}
}
@@ -543,9 +544,9 @@ static int remove_named_tab(DbTable *tb, int have_lock)
* We keep our increased refc over this op in order to
* prevent the table from disapearing.
*/
- erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+ db_unlock(tb, LCK_WRITE);
erts_smp_rwmtx_rwlock(rwlock);
- erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+ db_lock(tb, LCK_WRITE);
}
#endif
@@ -1417,12 +1418,12 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
{
DbTable init_tb;
- erts_smp_atomic_init(&init_tb.common.memory_size, 0);
+ erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
&init_tb, sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
- erts_smp_atomic_init(&tb->common.memory_size,
- erts_smp_atomic_read(&init_tb.common.memory_size));
+ erts_smp_atomic_init_nob(&tb->common.memory_size,
+ erts_smp_atomic_read_nob(&init_tb.common.memory_size));
}
tb->common.meth = meth;
@@ -1439,7 +1440,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
tb->common.owner = BIF_P->id;
set_heir(BIF_P, tb, heir, heir_data);
- erts_smp_atomic_init(&tb->common.nitems, 0);
+ erts_smp_atomic_init_nob(&tb->common.nitems, 0);
tb->common.fixations = NULL;
tb->common.compress = is_compressed;
@@ -1505,9 +1506,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
BIF_ARG_1, BIF_ARG_2, ret, BIF_P->id,
BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n",
- erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
+ erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n",
- erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
+ erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
#endif
UseTmpHeap(3,BIF_P);
@@ -1650,24 +1651,6 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE);
tb->common.status |= DB_DELETE;
- mmtl = get_meta_main_tab_lock(tb->common.slot);
-#ifdef ERTS_SMP
- if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
- /*
- * We keep our increased refc over this op in order to
- * prevent the table from disapearing.
- */
- erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
- erts_smp_rwmtx_rwlock(mmtl);
- erts_smp_rwmtx_rwlock(&tb->common.rwlock);
- }
-#endif
- /* We must keep the slot, to be found by db_proc_dead() if process dies */
- MARK_SLOT_DEAD(tb->common.slot);
- erts_smp_rwmtx_rwunlock(mmtl);
- if (is_atom(tb->common.id))
- remove_named_tab(tb, 0);
-
if (tb->common.owner != BIF_P->id) {
DeclareTmpHeap(meta_tuple,3,BIF_P);
@@ -1691,6 +1674,25 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
UnUseTmpHeap(3,BIF_P);
}
+
+ mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+ if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+ /*
+ * We keep our increased refc over this op in order to
+ * prevent the table from disapearing.
+ */
+ db_unlock(tb, LCK_WRITE);
+ erts_smp_rwmtx_rwlock(mmtl);
+ db_lock(tb, LCK_WRITE);
+ }
+#endif
+ /* We must keep the slot, to be found by db_proc_dead() if process dies */
+ MARK_SLOT_DEAD(tb->common.slot);
+ erts_smp_rwmtx_rwunlock(mmtl);
+ if (is_atom(tb->common.id))
+ remove_named_tab(tb, 0);
+
/* disable inheritance */
free_heir_data(tb);
tb->common.heir = am_none;
@@ -1995,7 +1997,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
BIF_ERROR(BIF_P, BADARG);
}
- nitems = erts_smp_atomic_read(&tb->common.nitems);
+ nitems = erts_smp_atomic_read_nob(&tb->common.nitems);
tb->common.meth->db_delete_all_objects(BIF_P, tb);
db_unlock(tb, LCK_WRITE);
BIF_RET(erts_make_integer(nitems,BIF_P));
@@ -2789,7 +2791,7 @@ void init_db(void)
}
#endif
- erts_smp_atomic_init(&erts_ets_misc_mem_size, 0);
+ erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0);
db_initialize_util();
if (user_requested_db_max_tabs < DB_DEF_MAX_TABS)
@@ -2831,13 +2833,13 @@ void init_db(void)
/*TT*/
/* Create meta table invertion. */
- erts_smp_atomic_init(&init_tb.common.memory_size, 0);
+ erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
&init_tb,
sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
- erts_smp_atomic_init(&meta_pid_to_tab->common.memory_size,
- erts_smp_atomic_read(&init_tb.common.memory_size));
+ erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size,
+ erts_smp_atomic_read_nob(&init_tb.common.memory_size));
meta_pid_to_tab->common.id = NIL;
meta_pid_to_tab->common.the_name = am_true;
@@ -2850,7 +2852,7 @@ void init_db(void)
#endif
meta_pid_to_tab->common.keypos = 1;
meta_pid_to_tab->common.owner = NIL;
- erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0);
+ erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0);
meta_pid_to_tab->common.slot = -1;
meta_pid_to_tab->common.meth = &db_hash;
meta_pid_to_tab->common.compress = 0;
@@ -2863,13 +2865,13 @@ void init_db(void)
erl_exit(1,"Unable to create ets metadata tables.");
}
- erts_smp_atomic_set(&init_tb.common.memory_size, 0);
+ erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0);
meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
&init_tb,
sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
- erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.memory_size,
- erts_smp_atomic_read(&init_tb.common.memory_size));
+ erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size,
+ erts_smp_atomic_read_nob(&init_tb.common.memory_size));
meta_pid_to_fixed_tab->common.id = NIL;
meta_pid_to_fixed_tab->common.the_name = am_true;
@@ -2882,7 +2884,7 @@ void init_db(void)
#endif
meta_pid_to_fixed_tab->common.keypos = 1;
meta_pid_to_fixed_tab->common.owner = NIL;
- erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0);
+ erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0);
meta_pid_to_fixed_tab->common.slot = -1;
meta_pid_to_fixed_tab->common.meth = &db_hash;
meta_pid_to_fixed_tab->common.compress = 0;
@@ -3421,7 +3423,7 @@ static void unfix_table_locked(Process* p, DbTable* tb,
unlocked:
if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
- && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) {
+ && erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) {
#ifdef ERTS_SMP
if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
/* Must have write lock while purging pseudo-deleted (OTP-8166) */
@@ -3606,7 +3608,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
Eterm ret = THE_NON_VALUE;
if (What == am_size) {
- ret = make_small(erts_smp_atomic_read(&tb->common.nitems));
+ ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems));
} else if (What == am_type) {
if (tb->common.status & DB_SET) {
ret = am_set;
@@ -3619,7 +3621,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
ret = am_bag;
}
} else if (What == am_memory) {
- Uint words = (Uint) ((erts_smp_atomic_read(&tb->common.memory_size)
+ Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
+ sizeof(Uint)
- 1)
/ sizeof(Uint));
@@ -3716,7 +3718,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
std_dev_exp = make_float(hp);
PUT_DOUBLE(f, hp);
hp += FLOAT_SIZE_OBJECT;
- ret = TUPLE6(hp, make_small(erts_smp_atomic_read(&tb->hash.nactive)),
+ ret = TUPLE6(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)),
avg, std_dev_real, std_dev_exp,
make_small(stats.min_chain_len),
make_small(stats.max_chain_len));
@@ -3735,9 +3737,9 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb)
tb->common.meth->db_print(to, to_arg, show, tb);
- erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems));
+ erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems));
erts_print(to, to_arg, "Words: %bpu\n",
- (UWord) ((erts_smp_atomic_read(&tb->common.memory_size)
+ (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
+ sizeof(Uint)
- 1)
/ sizeof(Uint)));
@@ -3763,8 +3765,9 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */
Uint
erts_get_ets_misc_mem_size(void)
{
+ ERTS_THR_MEMORY_BARRIER;
/* Memory not allocated in ets_alloc */
- return (Uint) erts_smp_atomic_read(&erts_ets_misc_mem_size);
+ return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
}
/* SMP Note: May only be used when system is locked */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index e0bdebcb01..2e5deaf338 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -86,11 +86,11 @@ do { \
erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \
- ((erts_aint_t) (FREE_SZ))); \
ASSERT((TAB)); \
- erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \
+ erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__); \
} while (0)
#define ERTS_ETS_MISC_MEM_ADD(SZ) \
- erts_smp_atomic_add(&erts_ets_misc_mem_size, (SZ));
+ erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ));
ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type,
DbTable *tab,
@@ -227,7 +227,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size)
ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0);
ASSERT(((void *) tab) != ptr
- || erts_smp_atomic_read(&tab->common.memory_size) == 0);
+ || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0);
erts_free(type, ptr);
}
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index fdc82c8b88..e3380a57b2 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -105,9 +105,13 @@
#define NSEG_2 256 /* Size of second segment table */
#define NSEG_INC 128 /* Number of segments to grow after that */
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab))
-#define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive))
-#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
+#ifdef ETHR_ORDERED_READ_DEPEND
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab))
+#else
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab))
+#endif
+#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
+#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK]
@@ -123,10 +127,10 @@
static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
{
Uint mask = erts_smp_atomic_read_acqb(&tb->szm);
- Uint ix = hval & mask;
- if (ix >= erts_smp_atomic_read(&tb->nactive)) {
+ Uint ix = hval & mask;
+ if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) {
ix &= mask>>1;
- ASSERT(ix < erts_smp_atomic_read(&tb->nactive));
+ ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive));
}
return ix;
}
@@ -141,14 +145,14 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
(DbTable *) tb,
sizeof(FixedDeletion));
ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
- fixd->slot = ix;
- was_next = erts_smp_atomic_read(&tb->fixdel);
+ fixd->slot = ix;
+ was_next = erts_smp_atomic_read_acqb(&tb->fixdel);
do { /* Lockless atomic insertion in linked list: */
exp_next = was_next;
fixd->next = (FixedDeletion*) exp_next;
- was_next = erts_smp_atomic_cmpxchg(&tb->fixdel,
- (erts_aint_t) fixd,
- exp_next);
+ was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+ (erts_aint_t) fixd,
+ exp_next);
}while (was_next != exp_next);
}
@@ -540,22 +544,24 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
{
/*int tries = 0;*/
DEBUG_WAIT();
- if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel,
- (erts_aint_t)NULL) != (erts_aint_t)NULL) {
+ if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+ (erts_aint_t) fixdel,
+ (erts_aint_t) NULL) != (erts_aint_t) NULL) {
/* Oboy, must join lists */
FixedDeletion* last = fixdel;
erts_aint_t was_tail;
erts_aint_t exp_tail;
- while (last->next != NULL) last = last->next;
- was_tail = erts_smp_atomic_read(&tb->fixdel);
+ while (last->next != NULL) last = last->next;
+ was_tail = erts_smp_atomic_read_acqb(&tb->fixdel);
do { /* Lockless atomic list insertion */
exp_tail = was_tail;
last->next = (FixedDeletion*) exp_tail;
/*++tries;*/
DEBUG_WAIT();
- was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel,
- exp_tail);
+ was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+ (erts_aint_t) fixdel,
+ exp_tail);
}while (was_tail != exp_tail);
}
/*erts_fprintf(stderr,"erl_db_hash: restore_fixdel tries=%d\r\n", tries);*/
@@ -572,7 +578,8 @@ void db_unfix_table_hash(DbTableHash *tb)
|| (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
&& !tb->common.is_thread_safe));
restart:
- fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL);
+ fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel,
+ (erts_aint_t) NULL);
while (fixdel != NULL) {
FixedDeletion *fx = fixdel;
int ix = fx->slot;
@@ -639,14 +646,14 @@ int db_create_hash(Process *p, DbTable *tbl)
{
DbTableHash *tb = &tbl->hash;
- erts_smp_atomic_init(&tb->szm, SEGSZ_MASK);
- erts_smp_atomic_init(&tb->nactive, SEGSZ);
- erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL);
- erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab);
+ erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK);
+ erts_smp_atomic_init_nob(&tb->nactive, SEGSZ);
+ erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
+ erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab);
tb->nsegs = NSEG_1;
tb->nslots = SEGSZ;
- erts_smp_atomic_init(&tb->is_resizing, 0);
+ erts_smp_atomic_init_nob(&tb->is_resizing, 0);
#ifdef ERTS_SMP
if (tb->common.type & DB_FINE_LOCKED) {
erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
@@ -663,7 +670,7 @@ int db_create_hash(Process *p, DbTable *tbl)
/* This important property is needed to guarantee that the buckets
* involved in a grow/shrink operation it protected by the same lock:
*/
- ASSERT(erts_smp_atomic_read(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
+ ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
}
else { /* coarse locking */
tb->locks = NULL;
@@ -783,7 +790,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
if (tb->common.status & DB_SET) {
HashDbTerm* bnext = b->next;
if (b->hvalue == INVALID_HASH) {
- erts_smp_atomic_inc(&tb->common.nitems);
+ erts_smp_atomic_inc_nob(&tb->common.nitems);
}
else if (key_clash_fail) {
ret = DB_ERROR_BADKEY;
@@ -811,7 +818,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
do {
if (db_eq(&tb->common,obj,&q->dbterm)) {
if (q->hvalue == INVALID_HASH) {
- erts_smp_atomic_inc(&tb->common.nitems);
+ erts_smp_atomic_inc_nob(&tb->common.nitems);
q->hvalue = hval;
if (q != b) { /* must move to preserve key insertion order */
*qp = q->next;
@@ -832,7 +839,7 @@ Lnew:
q->hvalue = hval;
q->next = b;
*bp = q;
- nitems = erts_smp_atomic_inctest(&tb->common.nitems);
+ nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
WUNLOCK_HASH(lck);
{
int nactive = NACTIVE(tb);
@@ -1069,7 +1076,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value)
EQ(value, b->dbterm.tpl[2])) {
*bp = b->next;
free_term(tb, b);
- erts_smp_atomic_dec(&tb->common.nitems);
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
b = *bp;
break;
}
@@ -1128,7 +1135,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
}
WUNLOCK_HASH(lck);
if (nitems_diff) {
- erts_smp_atomic_add(&tb->common.nitems, nitems_diff);
+ erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
try_shrink(tb);
}
*ret = am_true;
@@ -1187,7 +1194,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
}
WUNLOCK_HASH(lck);
if (nitems_diff) {
- erts_smp_atomic_add(&tb->common.nitems, nitems_diff);
+ erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
try_shrink(tb);
}
*ret = am_true;
@@ -1798,7 +1805,7 @@ static int db_select_delete_hash(Process *p,
free_term(tb, del);
did_erase = 1;
}
- erts_smp_atomic_dec(&tb->common.nitems);
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
++got;
}
--num_left;
@@ -1909,7 +1916,7 @@ static int db_select_delete_continue_hash(Process *p,
free_term(tb, del);
did_erase = 1;
}
- erts_smp_atomic_dec(&tb->common.nitems);
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
++got;
}
@@ -2064,7 +2071,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
}while(list != NULL);
}
}
- erts_smp_atomic_set(&tb->common.nitems, 0);
+ erts_smp_atomic_set_nob(&tb->common.nitems, 0);
return DB_ERROR_NONE;
}
@@ -2115,7 +2122,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
{
DbTableHash *tb = &tbl->hash;
int done;
- FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read(&tb->fixdel);
+ FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel);
ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb));
done = 0;
@@ -2129,11 +2136,11 @@ static int db_free_table_continue_hash(DbTable *tbl)
sizeof(FixedDeletion));
ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
if (++done >= 2*DELETE_RECORD_LIMIT) {
- erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel);
+ erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
return 0; /* Not done */
}
}
- erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL);
+ erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
done /= 2;
while(tb->nslots != 0) {
@@ -2157,7 +2164,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
tb->locks = NULL;
}
#endif
- ASSERT(erts_smp_atomic_read(&tb->common.memory_size) == sizeof(DbTable));
+ ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
return 1; /* Done */
}
@@ -2350,7 +2357,7 @@ static int alloc_seg(DbTableHash *tb)
struct ext_segment* eseg;
eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
- erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab);
+ erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) eseg->segtab);
tb->nsegs = eseg->nsegs;
}
ASSERT(seg_ix < tb->nsegs);
@@ -2422,7 +2429,7 @@ static int free_seg(DbTableHash *tb, int free_records)
MY_ASSERT(newtop->s.is_ext_segment);
if (newtop->prev_segtab != NULL) {
/* Time to use a smaller segtab */
- erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
+ erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
tb->nsegs = seg_ix;
ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
}
@@ -2439,7 +2446,7 @@ static int free_seg(DbTableHash *tb, int free_records)
if (seg_ix > 0) {
if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
} else {
- erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL);
+ erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t)NULL);
}
#endif
tb->nslots -= SEGSZ;
@@ -2500,7 +2507,7 @@ static void grow(DbTableHash* tb, int nactive)
int from_ix;
int szm;
- if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) {
+ if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) {
return; /* already in progress */
}
if (NACTIVE(tb) != nactive) {
@@ -2515,7 +2522,7 @@ static void grow(DbTableHash* tb, int nactive)
}
ASSERT(nactive < tb->nslots);
- szm = erts_smp_atomic_read(&tb->szm);
+ szm = erts_smp_atomic_read_nob(&tb->szm);
if (nactive <= szm) {
from_ix = nactive & (szm >> 1);
} else {
@@ -2532,7 +2539,7 @@ static void grow(DbTableHash* tb, int nactive)
WUNLOCK_HASH(lck);
goto abort;
}
- erts_smp_atomic_inc(&tb->nactive);
+ erts_smp_atomic_inc_nob(&tb->nactive);
if (from_ix == 0) {
erts_smp_atomic_set_relb(&tb->szm, szm);
}
@@ -2577,13 +2584,13 @@ abort:
*/
static void shrink(DbTableHash* tb, int nactive)
{
- if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) {
+ if (erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1)) {
return; /* already in progress */
}
if (NACTIVE(tb) == nactive) {
erts_smp_rwmtx_t* lck;
int src_ix = nactive - 1;
- int low_szm = erts_smp_atomic_read(&tb->szm) >> 1;
+ int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1;
int dst_ix = src_ix & low_szm;
ASSERT(dst_ix < src_ix);
@@ -2610,7 +2617,7 @@ static void shrink(DbTableHash* tb, int nactive)
*dst_bp = *src_bp;
*src_bp = NULL;
- erts_smp_atomic_set(&tb->nactive, src_ix);
+ erts_smp_atomic_set_nob(&tb->nactive, src_ix);
if (dst_ix == 0) {
erts_smp_atomic_set_relb(&tb->szm, low_szm);
}
@@ -2746,7 +2753,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
} else {
db_free_table_hash(tbl);
db_create_hash(p, tbl);
- erts_smp_atomic_set(&tbl->hash.common.nitems, 0);
+ erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0);
}
return 0;
}
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index 9a0ba3a418..c6f0d80e32 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -49,7 +49,7 @@
#include "erl_db_tree.h"
#define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos))
-#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
+#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
/*
** A stack of this size is enough for an AVL tree with more than
@@ -84,7 +84,7 @@
*/
static DbTreeStack* get_static_stack(DbTableTree* tb)
{
- if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) {
+ if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
return &tb->static_stack;
}
return NULL;
@@ -96,7 +96,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb)
static DbTreeStack* get_any_stack(DbTableTree* tb)
{
DbTreeStack* stack;
- if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) {
+ if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
return &tb->static_stack;
}
stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -110,7 +110,7 @@ static DbTreeStack* get_any_stack(DbTableTree* tb)
static void release_stack(DbTableTree* tb, DbTreeStack* stack)
{
if (stack == &tb->static_stack) {
- ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1);
+ ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1);
erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
}
else {
@@ -478,7 +478,7 @@ int db_create_tree(Process *p, DbTable *tbl)
sizeof(TreeDbTerm *) * STACK_NEED);
tb->static_stack.pos = 0;
tb->static_stack.slot = 0;
- erts_smp_atomic_init(&tb->is_stack_busy, 0);
+ erts_smp_atomic_init_nob(&tb->is_stack_busy, 0);
tb->deletion = 0;
return DB_ERROR_NONE;
}
@@ -613,8 +613,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
for (;;)
if (!*this) { /* Found our place */
state = 1;
- if (erts_smp_atomic_inctest(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
- erts_smp_atomic_dec(&tb->common.nitems);
+ if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
return DB_ERROR_SYSRES;
}
*this = new_dbterm(tb, obj);
@@ -1583,7 +1583,7 @@ static int db_select_delete_continue_tree(Process *p,
sc.max = 1000;
sc.keypos = tb->common.keypos;
- ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy));
+ ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy));
traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc);
BUMP_REDS(p, 1000 - sc.max);
@@ -1774,7 +1774,7 @@ static int db_free_table_continue_tree(DbTable *tbl)
(DbTable *) tb,
(void *) tb->static_stack.array,
sizeof(TreeDbTerm *) * STACK_NEED);
- ASSERT(erts_smp_atomic_read(&tb->common.memory_size)
+ ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size)
== sizeof(DbTable));
}
return result;
@@ -1784,7 +1784,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl)
{
db_free_table_tree(tbl);
db_create_tree(p, tbl);
- erts_smp_atomic_set(&tbl->tree.common.nitems, 0);
+ erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0);
return 0;
}
@@ -1866,7 +1866,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
tstack[tpos++] = this;
state = delsub(this);
}
- erts_smp_atomic_dec(&tb->common.nitems);
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
break;
}
}
@@ -1933,7 +1933,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
tstack[tpos++] = this;
state = delsub(this);
}
- erts_smp_atomic_dec(&tb->common.nitems);
+ erts_smp_atomic_dec_nob(&tb->common.nitems);
break;
}
}
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index c3b074f782..7dfbb2ed02 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -491,7 +491,7 @@ erts_match_set_release_result(Process* c_p)
/* The trace control word. */
-static erts_smp_atomic_t trace_control_word;
+static erts_smp_atomic32_t trace_control_word;
/* This needs to be here, before the bif table... */
@@ -911,7 +911,7 @@ static void db_free_tmp_uncompressed(DbTerm* obj);
BIF_RETTYPE db_get_trace_control_word_0(Process *p)
{
- Uint32 tcw = (Uint32) erts_smp_atomic_read(&trace_control_word);
+ Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word);
BIF_RET(erts_make_integer((Uint) tcw, p));
}
@@ -924,7 +924,8 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new)
if (val != ((Uint32)val))
BIF_ERROR(p, BADARG);
- old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val);
+ old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word,
+ (erts_aint32_t) val);
BIF_RET(erts_make_integer((Uint) old_tcw, p));
}
@@ -1249,7 +1250,7 @@ void db_initialize_util(void){
sizeof(DMCGuardBif),
(int (*)(const void *, const void *)) &cmp_guard_bif);
match_pseudo_process_init();
- erts_smp_atomic_init(&trace_control_word, 0);
+ erts_smp_atomic32_init_nob(&trace_control_word, 0);
}
@@ -1731,6 +1732,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
#define BEGIN_ATOMIC_TRACE(p) \
do { \
if (! atomic_trace) { \
+ erts_refc_inc(&bprog->refc, 2); \
erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \
erts_smp_block_system(0); \
atomic_trace = !0; \
@@ -1741,6 +1743,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog,
if (atomic_trace) { \
erts_smp_release_system(); \
erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \
+ if (erts_refc_dectest(&bprog->refc, 0) == 0) {\
+ erts_bin_free(bprog); \
+ } \
atomic_trace = 0; \
} \
} while (0)
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index 76b206d76f..1cc508ac5a 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix)
/* Prototypes of callback functions */
static Block_t * get_free_block (Allctr_t *, Uint,
- Block_t *, Uint);
-static void link_free_block (Allctr_t *, Block_t *);
-static void unlink_free_block (Allctr_t *, Block_t *);
-static void update_last_aux_mbc (Allctr_t *, Carrier_t *);
+ Block_t *, Uint, Uint32);
+static void link_free_block (Allctr_t *, Block_t *, Uint32);
+static void unlink_free_block (Allctr_t *, Block_t *, Uint32);
+static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32);
static Eterm info_options (Allctr_t *, char *, int *,
void *, Uint **, Uint *);
static void init_atoms (void);
@@ -197,6 +197,8 @@ erts_gfalc_start(GFAllctr_t *gfallctr,
is a struct). */
Allctr_t *allctr = (Allctr_t *) gfallctr;
+ init->sbmbct = 0; /* Small mbc not yet supported by goodfit */
+
sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t));
allctr->mbc_header_size = sizeof(Carrier_t);
@@ -379,7 +381,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
static Block_t *
get_free_block(Allctr_t *allctr, Uint size,
- Block_t *cand_blk, Uint cand_size)
+ Block_t *cand_blk, Uint cand_size, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
int unsafe_bi, min_bi;
@@ -398,7 +400,7 @@ get_free_block(Allctr_t *allctr, Uint size,
if (blk) {
if (cand_blk && cand_size <= BLK_SZ(blk))
return NULL; /* cand_blk was better */
- unlink_free_block(allctr, blk);
+ unlink_free_block(allctr, blk, flags);
return blk;
}
if (min_bi < NO_OF_BKTS - 1) {
@@ -418,14 +420,14 @@ get_free_block(Allctr_t *allctr, Uint size,
ASSERT(blk);
if (cand_blk && cand_size <= BLK_SZ(blk))
return NULL; /* cand_blk was better */
- unlink_free_block(allctr, blk);
+ unlink_free_block(allctr, blk, flags);
return blk;
}
static void
-link_free_block(Allctr_t *allctr, Block_t *block)
+link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
@@ -446,7 +448,7 @@ link_free_block(Allctr_t *allctr, Block_t *block)
}
static void
-unlink_free_block(Allctr_t *allctr, Block_t *block)
+unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
@@ -467,7 +469,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block)
}
static void
-update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc)
+update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags)
{
GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 0a57eb6d88..5f3f653e99 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -69,7 +69,8 @@ static void erl_init(int ncpu);
#define ERTS_MIN_COMPAT_REL 7
#ifdef ERTS_SMP
-erts_smp_atomic_t erts_writing_erl_crash_dump;
+erts_smp_atomic32_t erts_writing_erl_crash_dump;
+erts_tsd_key_t erts_is_crash_dumping_key;
#else
volatile int erts_writing_erl_crash_dump = 0;
#endif
@@ -323,7 +324,7 @@ init_shared_memory(int argc, char **argv)
#endif
global_gen_gcs = 0;
- global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
+ global_max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
global_gc_flags = erts_default_process_flags;
erts_global_offheap.mso = NULL;
@@ -646,12 +647,14 @@ early_init(int *argc, char **argv) /*
erts_lc_init();
#endif
#ifdef ERTS_SMP
- erts_smp_atomic_init(&erts_writing_erl_crash_dump, 0L);
+ erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
+ erts_tsd_key_create(&erts_is_crash_dumping_key);
#else
erts_writing_erl_crash_dump = 0;
#endif
- erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1));
+ erts_smp_atomic32_init_nob(&erts_max_gen_gcs,
+ (erts_aint32_t) ((Uint16) -1));
erts_pre_init_process();
#if defined(USE_THREADS) && !defined(ERTS_SMP)
@@ -856,7 +859,8 @@ erl_start(int argc, char **argv)
envbufsz = sizeof(envbuf);
if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) {
Uint16 max_gen_gcs = atoi(envbuf);
- erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs);
+ erts_smp_atomic32_set_nob(&erts_max_gen_gcs,
+ (erts_aint32_t) max_gen_gcs);
}
envbufsz = sizeof(envbuf);
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index f3f3c22933..04ea004ef7 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1186,6 +1186,8 @@ erts_instr_init(int stat, int map_stat)
sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1));
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+ if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+ continue;
if (erts_allctrs_info[i].enabled)
stats->ap[i] = &stats->a[i];
else
@@ -1199,6 +1201,8 @@ erts_instr_init(int stat, int map_stat)
erts_instr_memory_map = 1;
erts_instr_stat = 1;
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+ if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+ continue;
erts_allctrs[i].alloc = map_stat_alloc;
erts_allctrs[i].realloc = map_stat_realloc;
erts_allctrs[i].free = map_stat_free;
@@ -1209,6 +1213,8 @@ erts_instr_init(int stat, int map_stat)
else {
erts_instr_stat = 1;
for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+ if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+ continue;
erts_allctrs[i].alloc = stat_alloc;
erts_allctrs[i].realloc = stat_realloc;
erts_allctrs[i].free = stat_free;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 9180508a49..587d82f2bb 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -153,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "instr", NULL },
{ "fix_alloc", "index" },
{ "alcu_allocator", "index" },
+ { "sbmbc_alloc", "index" },
{ "alcu_delayed_free", "index" },
{ "mseg", NULL },
#if HALFWORD_HEAP
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index 9751b5d77c..47597f302b 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -125,7 +125,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name)
} else {
n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH,
mon_size*sizeof(Uint));
- erts_smp_atomic_add(&tot_link_lh_size, mon_size*sizeof(Uint));
+ erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
}
hp = n->heap;
@@ -156,7 +156,7 @@ static ErtsLink *create_link(Uint type, Eterm pid)
} else {
n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH,
lnk_size*sizeof(Uint));
- erts_smp_atomic_add(&tot_link_lh_size, lnk_size*sizeof(Uint));
+ erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
}
hp = n->heap;
@@ -191,13 +191,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid)
void
erts_init_monitors(void)
{
- erts_smp_atomic_init(&tot_link_lh_size, 0);
+ erts_smp_atomic_init_nob(&tot_link_lh_size, 0);
}
Uint
erts_tot_link_lh_size(void)
{
- return (Uint) erts_smp_atomic_read(&tot_link_lh_size);
+ return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size);
}
void erts_destroy_monitor(ErtsMonitor *mon)
@@ -222,7 +222,7 @@ void erts_destroy_monitor(ErtsMonitor *mon)
erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon);
} else {
erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon);
- erts_smp_atomic_add(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
+ erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
}
}
@@ -244,7 +244,7 @@ void erts_destroy_link(ErtsLink *lnk)
erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk);
} else {
erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk);
- erts_smp_atomic_add(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
+ erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
}
}
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 68421b4387..ea781a6cd0 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -833,8 +833,11 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i)
ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i)
{
+ if (IS_SSMALL(i)) {
+ return make_small(i);
+ }
#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
- return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2));
+ return small_to_big(i, alloc_heap(env,2));
#elif SIZEOF_LONG == 8
ensure_heap(env,3);
return erts_sint64_to_big(i, &env->hp);
@@ -843,8 +846,11 @@ ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i)
ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i)
{
+ if (IS_USMALL(0,i)) {
+ return make_small(i);
+ }
#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
- return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2));
+ return uint_to_big(i,alloc_heap(env,2));
#elif SIZEOF_LONG == 8
ensure_heap(env,3);
return erts_uint64_to_big(i, &env->hp);
@@ -1007,6 +1013,29 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size)
driver_system_info(sip, si_size);
}
+int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) {
+ Eterm *listptr, ret = NIL, *hp;
+
+ if (is_nil(term)) {
+ *list = term;
+ return 1;
+ }
+
+ ret = NIL;
+
+ while (is_not_nil(term)) {
+ if (is_not_list(term)) {
+ return 0;
+ }
+ hp = alloc_heap(env, 2);
+ listptr = list_val(term);
+ ret = CONS(hp, CAR(listptr), ret);
+ term = CDR(listptr);
+ }
+ *list = ret;
+ return 1;
+}
+
ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); }
void enif_mutex_destroy(ErlNifMutex *mtx) { erl_drv_mutex_destroy(mtx); }
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index d028567faf..fea527f954 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -32,9 +32,10 @@
** 2.0: R14A
** 2.1: R14B02 "vm_variant"
** 2.2: R14B03 enif_is_exception
+** 2.3: R15 enif_make_reverse_list
*/
#define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 2
+#define ERL_NIF_MINOR_VERSION 3
#include <stdlib.h>
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index c991b61abe..4af9f61000 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -136,6 +136,7 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64));
ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64));
#endif
ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list));
/*
** Add new entries here to keep compatibility on Windows!!!
@@ -256,6 +257,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term));
#endif
# define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception)
+# define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list)
/*
** Add new entries here
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 6daa127d23..af3873995e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -118,7 +118,7 @@ dist_table_alloc(void *dep_tmpl)
dep->finalized_out_queue.first = NULL;
dep->finalized_out_queue.last = NULL;
- erts_smp_atomic_init(&dep->dist_cmd_scheduled, 0);
+ erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0);
erts_port_task_handle_init(&dep->dist_cmd);
dep->send = NULL;
dep->cache = NULL;
@@ -767,7 +767,7 @@ void erts_init_node_tables(void)
erts_this_dist_entry->finalized_out_queue.first = NULL;
erts_this_dist_entry->finalized_out_queue.last = NULL;
- erts_smp_atomic_init(&erts_this_dist_entry->dist_cmd_scheduled, 0);
+ erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0);
erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd);
erts_this_dist_entry->send = NULL;
erts_this_dist_entry->cache = NULL;
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index e6b55c45e4..6aa5161b08 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -121,7 +121,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_taskq,
static ERTS_INLINE ErtsPortTask *
handle2task(ErtsPortTaskHandle *pthp)
{
- return (ErtsPortTask *) erts_smp_atomic_read(pthp);
+ return (ErtsPortTask *) erts_smp_atomic_read_nob(pthp);
}
static ERTS_INLINE void
@@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp)
{
if (ptp->handle) {
ASSERT(ptp == handle2task(ptp->handle));
- erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL);
+ erts_smp_atomic_set_nob(ptp->handle, (erts_aint_t) NULL);
}
}
@@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
{
ptp->handle = pthp;
if (pthp) {
- erts_smp_atomic_set(pthp, (erts_aint_t) ptp);
+ erts_smp_atomic_set_nob(pthp, (erts_aint_t) ptp);
ASSERT(ptp == handle2task(ptp->handle));
}
}
@@ -479,8 +479,8 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp)
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT:
case ERTS_PORT_TASK_EVENT:
- ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) > 0);
- erts_smp_atomic_dec(&erts_port_task_outstanding_io_tasks);
+ ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0);
+ erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
break;
default:
break;
@@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id,
ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
if (xrunq) {
/* Port emigrated ... */
- erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq);
+ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
erts_smp_runq_unlock(runq);
runq = xrunq;
}
@@ -594,7 +594,7 @@ erts_port_task_schedule(Eterm id,
case ERTS_PORT_TASK_INPUT:
case ERTS_PORT_TASK_OUTPUT:
case ERTS_PORT_TASK_EVENT:
- erts_smp_atomic_inc(&erts_port_task_outstanding_io_tasks);
+ erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
/* Fall through... */
default:
enqueue_task(pp->sched.taskq, ptp);
@@ -662,7 +662,7 @@ erts_port_task_free_port(Port *pp)
pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
erts_may_save_closed_port(pp);
erts_smp_port_state_unlock(pp);
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 1);
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1);
ptp->type = ERTS_PORT_TASK_FREE;
ptp->event = (ErlDrvEvent) -1;
ptp->event_data = NULL;
@@ -684,9 +684,9 @@ erts_port_task_free_port(Port *pp)
erts_may_save_closed_port(pp);
erts_smp_port_state_unlock(pp);
#ifdef ERTS_SMP
- erts_smp_atomic_dec(&pp->refc); /* Not alive */
+ erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
#endif
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
handle_remaining_tasks(runq, pp); /* May release runq lock */
ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first));
pp->sched.taskq = NULL;
@@ -724,7 +724,7 @@ resume_after_block(void *vd)
ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd;
erts_smp_runq_lock(d->runq);
if (d->resp)
- *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+ *d->resp = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
!= (erts_aint_t) 0);
}
@@ -832,8 +832,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
ASSERT(!ptqp->first
&& (!pp->sched.taskq || !pp->sched.taskq->first));
#ifdef ERTS_SMP
- erts_smp_atomic_dec(&pp->refc); /* Not alive */
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */
+ erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
#else
erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE);
#endif
@@ -906,14 +906,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
erts_unblock_fpe(fpe_was_unmasked);
if (io_tasks_executed) {
- ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed);
- erts_smp_atomic_add(&erts_port_task_outstanding_io_tasks, -1*io_tasks_executed);
+ ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+ >= io_tasks_executed);
+ erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
+ -1*io_tasks_executed);
}
*curr_port_pp = NULL;
#ifdef ERTS_SMP
- ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read(&pp->run_queue));
+ ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
#endif
if (!pp->sched.taskq) {
@@ -940,7 +942,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
}
else {
/* Port emigrated ... */
- erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq);
+ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
enqueue_port(xrunq, pp);
ASSERT(pp->sched.exe_taskq);
pp->sched.exe_taskq = NULL;
@@ -951,7 +953,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
port_was_enqueued = 1;
}
- res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+ res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
!= (erts_aint_t) 0);
ERTS_PT_CHK_PRES_PORTQ(runq, pp);
@@ -972,13 +974,13 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
{
erts_aint_t refc;
erts_smp_mtx_unlock(pp->lock);
- refc = erts_smp_atomic_dectest(&pp->refc);
+ refc = erts_smp_atomic_dec_read_nob(&pp->refc);
ASSERT(refc >= 0);
if (refc == 0) {
erts_smp_runq_unlock(runq);
erts_port_cleanup(pp); /* Might aquire runq lock */
erts_smp_runq_lock(runq);
- res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks)
+ res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
!= (erts_aint_t) 0);
}
}
@@ -1107,12 +1109,12 @@ erts_port_migrate(Port *prt, int *prt_locked,
/* Refuse to migrate to a suspended run queue */
if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
- if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue))
+ if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue))
return ERTS_MIGRATE_FAILED_RUNQ_CHANGED;
if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt))
return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
dequeue_port(from_rq, prt);
- erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq);
+ erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq);
enqueue_port(to_rq, prt);
return ERTS_MIGRATE_SUCCESS;
}
@@ -1125,7 +1127,8 @@ erts_port_migrate(Port *prt, int *prt_locked,
void
erts_port_task_init(void)
{
- erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0);
+ erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
+ (erts_aint_t) 0);
init_port_task_alloc();
init_port_taskq_alloc();
}
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 3e2c5f07ab..d7104e1143 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -79,13 +79,13 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
ERTS_GLB_INLINE void
erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
{
- erts_smp_atomic_init(pthp, (erts_aint_t) NULL);
+ erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL);
}
ERTS_GLB_INLINE int
erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
{
- return ((void *) erts_smp_atomic_read(pthp)) != NULL;
+ return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL;
}
ERTS_GLB_INLINE void
@@ -102,8 +102,8 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp)
ERTS_GLB_INLINE int
erts_port_task_have_outstanding_io_tasks(void)
{
- ERTS_THR_MEMORY_BARRIER;
- return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0;
+ return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
+ != 0);
}
#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 2704359a8f..bbdcf79d00 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -134,15 +134,15 @@ int erts_disable_proc_not_running_opt;
#ifndef DEBUG
#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
- erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL))
+ erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL))
#else
#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
do { \
erts_aint32_t old_val__; \
- old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \
- (VAL)); \
+ old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \
+ (VAL)); \
ASSERT(old_val__ == (OLD_VAL)); \
} while (0)
@@ -158,7 +158,7 @@ static struct {
erts_smp_atomic32_t changing;
erts_smp_atomic32_t active;
struct {
- erts_smp_atomic32_t ongoing;
+ int ongoing;
long wait_active;
ErtsProcList *procs;
} msb; /* Multi Scheduling Block */
@@ -410,7 +410,7 @@ erts_init_process(int ncpu)
init_proclist_alloc();
- erts_smp_atomic32_init(&process_count, 0);
+ erts_smp_atomic32_init_nob(&process_count, 0);
if (erts_use_r9_pids_ports) {
proc_bits = ERTS_R9_PROC_BITS;
@@ -694,8 +694,8 @@ erts_smp_schedule_misc_aux_work(int ignore_self,
erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx);
ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
- aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
- ERTS_SSI_AUX_WORK_MISC);
+ aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work,
+ ERTS_SSI_AUX_WORK_MISC);
if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0)
erts_sched_poke(ssi);
}
@@ -711,8 +711,8 @@ erts_smp_notify_check_children_needed(void)
erts_aint32_t aux_work;
ErtsSchedulerSleepInfo *ssi;
ssi = ERTS_SCHED_SLEEP_INFO_IX(i);
- aux_work = erts_smp_atomic32_bor(&ssi->aux_work,
- ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+ aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work,
+ ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN))
erts_sched_poke(ssi);
}
@@ -727,15 +727,15 @@ blockable_aux_work(ErtsSchedulerData *esdp,
{
if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
- aux_work = erts_smp_atomic32_band(&ssi->aux_work,
- ~ERTS_SSI_AUX_WORK_MISC);
+ aux_work = erts_smp_atomic32_read_band_nob(&ssi->aux_work,
+ ~ERTS_SSI_AUX_WORK_MISC);
aux_work &= ~ERTS_SSI_AUX_WORK_MISC;
handle_misc_aux_work(esdp);
}
#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
- aux_work = erts_smp_atomic32_band(&ssi->aux_work,
- ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+ aux_work = erts_smp_atomic32_band_nob(&ssi->aux_work,
+ ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
erts_check_children();
}
@@ -815,7 +815,7 @@ erts_active_schedulers(void)
static ERTS_INLINE void
clear_sys_scheduling(void)
{
- erts_smp_atomic32_set_relb(&doing_sys_schedule, 0);
+ erts_smp_atomic32_set_mb(&doing_sys_schedule, 0);
}
static ERTS_INLINE int
@@ -882,42 +882,43 @@ sched_active(Uint no, ErtsRunQueue *rq)
static int ERTS_INLINE
ongoing_multi_scheduling_block(void)
{
- return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0;
+ ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx));
+ return schdlr_sspnd.msb.ongoing;
}
static ERTS_INLINE void
empty_runq(ErtsRunQueue *rq)
{
- erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags,
- ~ERTS_RUNQ_IFLG_NONEMPTY);
+ erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags,
+ ~ERTS_RUNQ_IFLG_NONEMPTY);
if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) {
#ifdef DEBUG
- erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
+ erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
/*
* For a short period of time no_empty_run_queues may have
* been increased twice for a specific run queue.
*/
ASSERT(0 <= empty && empty < 2*erts_no_run_queues);
#endif
- erts_smp_atomic32_inc(&no_empty_run_queues);
+ erts_smp_atomic32_inc_relb(&no_empty_run_queues);
}
}
static ERTS_INLINE void
non_empty_runq(ErtsRunQueue *rq)
{
- erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags,
- ERTS_RUNQ_IFLG_NONEMPTY);
+ erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags,
+ ERTS_RUNQ_IFLG_NONEMPTY);
if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) {
#ifdef DEBUG
- erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues);
+ erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
/*
* For a short period of time no_empty_run_queues may have
* been increased twice for a specific run queue.
*/
ASSERT(0 < empty && empty <= 2*erts_no_run_queues);
#endif
- erts_smp_atomic32_dec(&no_empty_run_queues);
+ erts_smp_atomic32_dec_relb(&no_empty_run_queues);
}
}
@@ -930,7 +931,7 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
erts_aint32_t xflgs = 0;
do {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return nflgs;
xflgs = oflgs;
@@ -947,7 +948,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING;
do {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return nflgs;
xflgs = oflgs;
@@ -989,7 +990,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
erts_tse_reset(ssi->event);
while (1) {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return nflgs;
if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
@@ -1049,7 +1050,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
tse_wait:
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
tse_blockable_aux_work:
aux_work = blockable_aux_work(esdp, ssi, aux_work);
#endif
@@ -1059,7 +1060,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
#endif
nonblockable_aux_work(esdp, ssi, aux_work);
#endif
@@ -1092,7 +1093,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
}
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
goto tse_blockable_aux_work;
@@ -1104,7 +1105,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
- erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
erts_smp_runq_lock(rq);
sched_active(esdp->no, rq);
@@ -1136,12 +1137,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
sys_aux_work:
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
aux_work = blockable_aux_work(esdp, ssi, aux_work);
#endif
#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK
#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
#endif
nonblockable_aux_work(esdp, ssi, aux_work);
#endif
@@ -1239,7 +1240,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
sys_locked_woken:
clear_sys_scheduling();
if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
- erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
sched_active_sys(esdp->no, rq);
}
}
@@ -1255,7 +1256,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
erts_aint32_t nflgs = 0;
erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
while (1) {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return oflgs;
nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
@@ -1298,7 +1299,6 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one)
erts_smp_spin_unlock(&sl->lock);
- ERTS_THR_MEMORY_BARRIER;
flgs = ssi_flags_set_wake(ssi);
erts_sched_finish_poke(ssi, flgs);
@@ -1344,13 +1344,13 @@ init_no_runqs(int active, int used)
{
erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
- erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs);
+ erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs);
}
static ERTS_INLINE void
get_no_runqs(int *active, int *used)
{
- erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs);
+ erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
if (active)
*active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
if (used)
@@ -1360,11 +1360,12 @@ get_no_runqs(int *active, int *used)
static ERTS_INLINE void
set_no_used_runqs(int used)
{
- erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
while (1) {
erts_aint32_t act, new;
- new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK);
- act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT;
+ new |= exp & ERTS_NO_RUNQS_MASK;
+ act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
if (act == exp)
break;
exp = act;
@@ -1374,11 +1375,12 @@ set_no_used_runqs(int used)
static ERTS_INLINE void
set_no_active_runqs(int active)
{
- erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
while (1) {
erts_aint32_t act, new;
- new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active;
- act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
+ new |= active & ERTS_NO_RUNQS_MASK;
+ act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
if (act == exp)
break;
exp = act;
@@ -1388,13 +1390,14 @@ set_no_active_runqs(int active)
static ERTS_INLINE int
try_inc_no_active_runqs(int active)
{
- erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+ erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
return 0;
if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
erts_aint32_t new, act;
- new = (exp & ~ERTS_NO_RUNQS_MASK) | active;
- act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+ new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
+ new |= active & ERTS_NO_RUNQS_MASK;
+ act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
if (act == exp)
return 1;
}
@@ -1410,7 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
if (crq->ix == ix)
return 0;
wrq = ERTS_RUNQ_IX(ix);
- iflgs = erts_smp_atomic32_read(&wrq->info_flags);
+ iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags);
if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
if (activate) {
if (try_inc_no_active_runqs(ix+1)) {
@@ -1652,15 +1655,15 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
erts_smp_runq_lock(evac_rq);
- erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
| ERTS_RUNQ_FLGS_EVACUATE_QMASK
| ERTS_RUNQ_FLG_SUSPENDED);
- erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
+ erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
/*
* Need to set up evacuation paths first since we
* may release the run queue lock on evac_rq
@@ -1909,7 +1912,7 @@ static ERTS_INLINE int
check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
{
ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
- erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags);
+ erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags);
if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY)
return try_steal_task_from_victim(rq, rq_lockedp, vrq);
else
@@ -2061,7 +2064,7 @@ check_balance(ErtsRunQueue *c_rq)
int forced, active, current_active, oowc, half_full_scheds, full_scheds,
mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix;
- if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) {
+ if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) {
c_rq->check_balance_reds = INT_MAX;
return;
}
@@ -2069,7 +2072,7 @@ check_balance(ErtsRunQueue *c_rq)
get_no_runqs(NULL, &blnc_no_rqs);
if (blnc_no_rqs == 1) {
c_rq->check_balance_reds = INT_MAX;
- erts_smp_atomic32_set(&balance_info.checking_balance, 0);
+ erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
return;
}
@@ -2077,7 +2080,7 @@ check_balance(ErtsRunQueue *c_rq)
if (balance_info.halftime) {
balance_info.halftime = 0;
- erts_smp_atomic32_set(&balance_info.checking_balance, 0);
+ erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
ERTS_FOREACH_RUNQ(rq,
{
if (rq->waiting)
@@ -2111,7 +2114,7 @@ check_balance(ErtsRunQueue *c_rq)
erts_smp_mtx_unlock(&balance_info.update_mtx);
erts_smp_runq_lock(c_rq);
c_rq->check_balance_reds = INT_MAX;
- erts_smp_atomic32_set(&balance_info.checking_balance, 0);
+ erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
return;
}
@@ -2456,7 +2459,7 @@ erts_fprintf(stderr, "--------------------------------\n");
set_no_active_runqs(active);
balance_info.halftime = 1;
- erts_smp_atomic32_set(&balance_info.checking_balance, 0);
+ erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
/* Write migration paths and reset balance statistics in all queues */
for (qix = 0; qix < blnc_no_rqs; qix++) {
@@ -2598,7 +2601,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS,
sizeof(ErtsAlignedRunQueue) * n);
#ifdef ERTS_SMP
- erts_smp_atomic32_init(&no_empty_run_queues, 0);
+ erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
#endif
erts_no_run_queues = n;
@@ -2608,7 +2611,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
rq->ix = ix;
- erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
+ erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
/* make sure that the "extra" id correponds to the schedulers
* id if the esdp->no <-> ix+1 mapping change.
@@ -2701,9 +2704,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
ssi->next = NULL;
ssi->prev = NULL;
#endif
- erts_smp_atomic32_init(&ssi->flags, 0);
+ erts_smp_atomic32_init_nob(&ssi->flags, 0);
ssi->event = NULL; /* initialized in sched_thread_func */
- erts_smp_atomic32_init(&ssi->aux_work, 0);
+ erts_smp_atomic32_init_nob(&ssi->aux_work, 0);
}
#endif
@@ -2747,7 +2750,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
}
#ifdef ERTS_SMP
- erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0);
+ erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0);
#endif
}
@@ -2755,11 +2758,11 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
erts_smp_cnd_init(&schdlr_sspnd.cnd);
- erts_smp_atomic32_init(&schdlr_sspnd.changing, 0);
+ erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
schdlr_sspnd.online = no_schedulers_online;
schdlr_sspnd.curr_online = no_schedulers;
- erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0);
- erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers);
+ schdlr_sspnd.msb.ongoing = 0;
+ erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers);
schdlr_sspnd.msb.procs = NULL;
init_no_runqs(no_schedulers,
erts_common_run_queue ? 1 : no_schedulers_online);
@@ -2768,7 +2771,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
balance_info.forced_check_balance = 0;
balance_info.halftime = 1;
balance_info.full_reds_history_index = 0;
- erts_smp_atomic32_init(&balance_info.checking_balance, 0);
+ erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0);
balance_info.prev_rise.active_runqs = 0;
balance_info.prev_rise.max_len = 0;
balance_info.prev_rise.reds = 0;
@@ -2777,8 +2780,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
if (no_schedulers_online < no_schedulers) {
if (erts_common_run_queue) {
for (ix = no_schedulers_online; ix < no_schedulers; ix++)
- erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
- ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+ ERTS_SSI_FLG_SUSPENDED);
}
else {
for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
@@ -2792,7 +2795,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
- erts_smp_atomic32_init(&doing_sys_schedule, 0);
+ erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
init_misc_aux_work();
@@ -2808,7 +2811,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
erts_no_schedulers = 1;
#endif
- erts_smp_atomic32_init(&function_calls, 0);
+ erts_smp_atomic32_init_nob(&function_calls, 0);
/* init port tasks */
erts_port_task_init();
@@ -2935,10 +2938,10 @@ int
erts_get_max_no_executing_schedulers(void)
{
#ifdef ERTS_SMP
- if (erts_smp_atomic32_read(&schdlr_sspnd.changing))
+ if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
return (int) erts_no_schedulers;
ERTS_THR_MEMORY_BARRIER;
- return (int) erts_smp_atomic32_read(&schdlr_sspnd.active);
+ return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
#else
return 1;
#endif
@@ -2968,7 +2971,7 @@ scheduler_ix_resume_wake(Uint ix)
| ERTS_SSI_FLG_SUSPENDED);
erts_aint32_t oflgs;
do {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs);
if (oflgs == xflgs) {
erts_sched_finish_poke(ssi, oflgs);
break;
@@ -2987,7 +2990,7 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
erts_aint32_t xflgs = xpct;
do {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return nflgs;
xflgs = oflgs;
@@ -3037,7 +3040,7 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
erts_tse_reset(ssi->event);
while (1) {
- oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs);
+ oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
if (oflgs == xflgs)
return nflgs;
if ((oflgs & (ERTS_SSI_FLG_SLEEPING
@@ -3092,15 +3095,15 @@ suspend_scheduler(ErtsSchedulerData *esdp)
flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
if (flgs & ERTS_SSI_FLG_SUSPENDED) {
- active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active);
+ active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
ASSERT(active_schedulers >= 1);
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
if (active_schedulers == schdlr_sspnd.msb.wait_active)
wake = 1;
if (active_schedulers == 1) {
- changing = erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+ changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_MSB);
changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
}
}
@@ -3122,8 +3125,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
&& schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
wake = 1;
if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
- changing = erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+ changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
}
}
@@ -3140,7 +3143,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
blockable_aux_work:
blockable_aux_work(esdp, ssi, aux_work);
#endif
@@ -3176,13 +3179,13 @@ suspend_scheduler(ErtsSchedulerData *esdp)
| ERTS_SSI_FLG_SUSPENDED));
if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
break;
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
break;
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
- aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) {
erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
goto blockable_aux_work;
@@ -3194,19 +3197,19 @@ suspend_scheduler(ErtsSchedulerData *esdp)
erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
}
- active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active);
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
&& schdlr_sspnd.online == active_schedulers) {
- erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+ erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_MSB);
}
ASSERT(no <= schdlr_sspnd.online);
- ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing));
+ ASSERT(!ongoing_multi_scheduling_block());
}
@@ -3235,7 +3238,7 @@ do { \
(RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \
| ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \
(RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \
- erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\
+ erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\
for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \
(RQ)->procs.prio_info[pix__].max_len = 0; \
(RQ)->procs.prio_info[pix__].reds = 0; \
@@ -3279,7 +3282,7 @@ erts_schedulers_state(Uint *total,
int res;
erts_aint32_t changing;
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
else {
@@ -3310,7 +3313,7 @@ erts_set_schedulers_online(Process *p,
have_unlocked_plocks = 0;
no = (int) new_no;
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if (changing) {
res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
}
@@ -3385,8 +3388,8 @@ erts_set_schedulers_online(Process *p,
for (ix = no; ix < online; ix++) {
ErtsSchedulerSleepInfo *ssi;
ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
- erts_smp_atomic32_bor(&ssi->flags,
- ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_bor_nob(&ssi->flags,
+ ERTS_SSI_FLG_SUSPENDED);
}
wake_all_schedulers();
}
@@ -3433,11 +3436,11 @@ erts_set_schedulers_online(Process *p,
NULL);
ASSERT(res != ERTS_SCHDLR_SSPND_DONE
? (ERTS_SCHDLR_SSPND_CHNG_WAITER
- & erts_smp_atomic32_read(&schdlr_sspnd.changing))
+ & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
: (ERTS_SCHDLR_SSPND_CHNG_WAITER
- == erts_smp_atomic32_read(&schdlr_sspnd.changing)));
- erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+ == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
+ erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
}
}
@@ -3456,7 +3459,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
ErtsProcList *plp;
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- changing = erts_smp_atomic32_read(&schdlr_sspnd.changing);
+ changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
if (changing) {
res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
}
@@ -3466,7 +3469,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
plp->next = schdlr_sspnd.msb.procs;
schdlr_sspnd.msb.procs = plp;
p->flags |= F_HAVE_BLCKD_MSCHED;
- ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
+ ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
ASSERT(p->scheduler_data->no == 1);
res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
}
@@ -3477,11 +3480,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
have_unlocked_plocks = 1;
erts_smp_proc_unlock(p, plocks);
}
- ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing));
- erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1);
+ ASSERT(!ongoing_multi_scheduling_block());
+ schdlr_sspnd.msb.ongoing = 1;
if (online == 1) {
res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
- ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
+ ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
ASSERT(p->scheduler_data->no == 1);
}
else {
@@ -3501,8 +3504,8 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
}
if (erts_common_run_queue) {
for (ix = 1; ix < online; ix++)
- erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
- ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+ ERTS_SSI_FLG_SUSPENDED);
wake_all_schedulers();
}
else {
@@ -3530,7 +3533,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
susp_sched_prep_block,
susp_sched_resume_block,
NULL);
- while (erts_smp_atomic32_read(&schdlr_sspnd.active)
+ while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
!= schdlr_sspnd.msb.wait_active)
erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
@@ -3539,11 +3542,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
NULL);
ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
? (ERTS_SCHDLR_SSPND_CHNG_WAITER
- & erts_smp_atomic32_read(&schdlr_sspnd.changing))
+ & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
: (ERTS_SCHDLR_SSPND_CHNG_WAITER
- == erts_smp_atomic32_read(&schdlr_sspnd.changing)));
- erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+ == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
+ erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
}
plp = proclist_create(p);
plp->next = schdlr_sspnd.msb.procs;
@@ -3610,16 +3613,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
});
#endif
p->flags &= ~F_HAVE_BLCKD_MSCHED;
- erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0);
+ schdlr_sspnd.msb.ongoing = 0;
if (schdlr_sspnd.online == 1) {
/* No schedulers to resume */
- ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1);
+ ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
}
else if (erts_common_run_queue) {
for (ix = 1; ix < schdlr_sspnd.online; ix++)
- erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
- ~ERTS_SSI_FLG_SUSPENDED);
+ erts_smp_atomic32_read_band_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags,
+ ~ERTS_SSI_FLG_SUSPENDED);
wake_all_schedulers();
}
else {
@@ -3669,7 +3672,7 @@ void
erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
{
if (return_value == am_blocked) {
- erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active);
+ erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
ASSERT(1 <= active && active <= 2);
ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1);
}
@@ -3752,12 +3755,12 @@ sched_thread_func(void *vesdp)
erts_thread_init_float();
erts_smp_mtx_lock(&schdlr_sspnd.mtx);
- ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing)
+ ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)
& ERTS_SCHDLR_SSPND_CHNG_ONLN);
if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) {
- erts_smp_atomic32_band(&schdlr_sspnd.changing,
- ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+ erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+ ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
if (((ErtsSchedulerData *) vesdp)->no != 1)
erts_smp_cnd_signal(&schdlr_sspnd.cnd);
}
@@ -5214,7 +5217,7 @@ Process *schedule(Process *p, int calls)
reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
esdp->virtual_reds = 0;
- fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds);
+ fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds);
ASSERT(esdp && esdp == erts_get_scheduler_data());
rq = erts_get_runq_current(esdp);
@@ -5349,7 +5352,7 @@ Process *schedule(Process *p, int calls)
if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
|| (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED)) {
- ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
+ ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED);
suspend_scheduler(esdp);
}
@@ -5363,7 +5366,7 @@ Process *schedule(Process *p, int calls)
|| defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK)
{
ErtsSchedulerSleepInfo *ssi = esdp->ssi;
- erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work);
+ erts_aint32_t aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work);
if (aux_work) {
erts_smp_runq_unlock(rq);
#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK
@@ -5407,7 +5410,7 @@ Process *schedule(Process *p, int calls)
if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
|| (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED)) {
- ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
+ ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
& ERTS_SSI_FLG_SUSPENDED);
non_empty_runq(rq);
goto continue_check_activities_to_run;
@@ -5948,7 +5951,7 @@ erts_test_next_pid(int set, Uint next)
Uint erts_process_count(void)
{
- erts_aint32_t res = erts_smp_atomic32_read(&process_count);
+ erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count);
ASSERT(res >= 0);
return (Uint) res;
}
@@ -5997,7 +6000,7 @@ alloc_process(void)
ASSERT(!process_tab[p_next]);
process_tab[p_next] = p;
- erts_smp_atomic32_inc(&process_count);
+ erts_smp_atomic32_inc_nob(&process_count);
p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
if (p->id == ERTS_INVALID_PID) {
/* Do not use the invalid pid; change serial */
@@ -6123,7 +6126,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
p->min_heap_size = H_MIN_SIZE;
p->min_vheap_size = BIN_VH_MIN_SIZE;
p->prio = PRIORITY_NORMAL;
- p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs);
+ p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
}
p->skipped = 0;
ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
@@ -7580,8 +7583,8 @@ continue_exit_process(Process *p
p->status_flags = 0;
#endif
process_tab[pix] = NULL; /* Time of death! */
- ASSERT(erts_smp_atomic32_read(&process_count) > 0);
- erts_smp_atomic32_dec(&process_count);
+ ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0);
+ erts_smp_atomic32_dec_nob(&process_count);
#ifdef ERTS_SMP
erts_pix_unlock(pix_lock);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 296acc7367..739aef3130 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1599,11 +1599,11 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
erts_aint32_t flags;
ERTS_THR_MEMORY_BARRIER;
- flags = erts_smp_atomic32_read(&ssi->flags);
+ flags = erts_smp_atomic32_read_nob(&ssi->flags);
ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING)
|| (flags & ERTS_SSI_FLG_WAITING));
if (flags & ERTS_SSI_FLG_SLEEPING) {
- flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
+ flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
erts_sched_finish_poke(ssi, flags);
}
}
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 5410bcd495..3550f1396c 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -350,7 +350,7 @@ heap_dump(int to, void *to_arg, Eterm x)
ProcBin* pb = (ProcBin *) binary_val(x);
Binary* val = pb->val;
- if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
+ if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) {
val->flags = (UWord) all_binaries;
all_binaries = val;
}
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 72560aa124..83379d7352 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -316,7 +316,7 @@ try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr)
break;
}
wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT;
- old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_(lck, wflg | lock);
+ old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock);
if (old_lflgs & lock) {
/* Didn't get the lock */
goto enqueue;
@@ -413,7 +413,7 @@ transfer_locks(Process *p,
do {
erts_tse_t *tmp = wake;
wake = wake->next;
- erts_atomic32_set(&tmp->uaflgs, 0);
+ erts_atomic32_set_nob(&tmp->uaflgs, 0);
erts_tse_set(tmp);
} while (wake);
@@ -509,14 +509,14 @@ wait_for_locks(Process *p,
ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
- erts_atomic32_set(&wtr->uaflgs, 1);
+ erts_atomic32_set_nob(&wtr->uaflgs, 1);
erts_pix_unlock(pix_lock);
while (1) {
int res;
erts_tse_reset(wtr);
- if (erts_atomic32_read(&wtr->uaflgs) == 0)
+ if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
break;
/*
@@ -955,7 +955,8 @@ erts_proc_lock_init(Process *p)
{
/* We always start with all locks locked */
#if ERTS_PROC_LOCK_ATOMIC_IMPL
- erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
+ erts_smp_atomic32_init_nob(&p->lock.flags,
+ (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
#else
p->lock.flags = ERTS_PROC_LOCKS_ALL;
#endif
@@ -974,7 +975,7 @@ erts_proc_lock_init(Process *p)
{
int i;
for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
- erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1);
+ erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
}
#endif
}
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 355179f084..cd3b2182fd 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -41,10 +41,10 @@
#define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
#define ERTS_PROC_LOCK_MUTEX_IMPL 0
-#if defined(ETHR_HAVE_OPTIMIZED_ATOMIC_OPS)
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# undef ERTS_PROC_LOCK_ATOMIC_IMPL
# define ERTS_PROC_LOCK_ATOMIC_IMPL 1
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCK)
+#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS)
# undef ERTS_PROC_LOCK_SPINLOCK_IMPL
# define ERTS_PROC_LOCK_SPINLOCK_IMPL 1
#else
@@ -270,9 +270,11 @@ typedef struct {
#if ERTS_PROC_LOCK_ATOMIC_IMPL
#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
- ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK)))
-#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \
- ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK)))
+ ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \
+ (erts_aint32_t) (MSK)))
+#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \
+ ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \
+ (erts_aint32_t) (MSK)))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
(erts_aint32_t) (NEW), \
@@ -282,7 +284,7 @@ typedef struct {
(erts_aint32_t) (NEW), \
(erts_aint32_t) (EXPECTED)))
#define ERTS_PROC_LOCK_FLGS_READ_(L) \
- ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags))
+ ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags))
#else /* no opt atomic ops */
@@ -325,7 +327,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
#endif
#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK))
-#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
+#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
@@ -623,11 +625,11 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
if (locks & lock) {
erts_aint32_t lock_count;
if (locked) {
- lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]);
+ lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]);
ERTS_LC_ASSERT(lock_count == 1);
}
else {
- lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]);
+ lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]);
ERTS_LC_ASSERT(lock_count == 0);
}
}
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 287327bfe1..a89ddfbcc1 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -54,8 +54,9 @@ typedef erts_cnd_t erts_smp_cnd_t;
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
-typedef erts_atomic_t erts_smp_atomic_t;
-typedef erts_atomic32_t erts_smp_atomic32_t;
+#define erts_smp_dw_atomic_t erts_dw_atomic_t
+#define erts_smp_atomic_t erts_atomic_t
+#define erts_smp_atomic32_t erts_atomic32_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
@@ -83,8 +84,9 @@ typedef struct {
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
-typedef SWord erts_smp_atomic_t;
-typedef Uint32 erts_smp_atomic32_t;
+#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
+#define erts_smp_atomic_t erts_no_atomic_t
+#define erts_smp_atomic32_t erts_no_atomic32_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
@@ -160,82 +162,6 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read(erts_smp_atomic32_t *var);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_inc(erts_smp_atomic32_t *incp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
char *name,
Eterm extra);
@@ -279,6 +205,429 @@ ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
+/*
+ * Functions implementing atomic operations with with no (nob),
+ * full (mb), acquire (acqb), release (relb), read (rb), and
+ * write (wb) memory barriers.
+ *
+ * If SMP support has been disabled, they are mapped to functions
+ * that performs the same operation, but aren't atomic and don't
+ * imply memory barriers.
+ */
+
+#ifdef ERTS_SMP
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
+#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
+#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
+#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob
+
+#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
+#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
+#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
+#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb
+
+#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
+#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
+#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb
+
+#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
+#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
+#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
+#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
+
+#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
+#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
+#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
+#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb
+
+#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
+#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
+#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
+#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_atomic_init_nob
+#define erts_smp_atomic_set_nob erts_atomic_set_nob
+#define erts_smp_atomic_read_nob erts_atomic_read_nob
+#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
+#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
+#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
+#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
+#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
+#define erts_smp_atomic_add_nob erts_atomic_add_nob
+#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
+#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
+#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
+#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
+
+#define erts_smp_atomic_init_mb erts_atomic_init_mb
+#define erts_smp_atomic_set_mb erts_atomic_set_mb
+#define erts_smp_atomic_read_mb erts_atomic_read_mb
+#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
+#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
+#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
+#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
+#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
+#define erts_smp_atomic_add_mb erts_atomic_add_mb
+#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
+#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
+#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
+#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
+
+#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
+#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
+#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
+#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
+#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
+#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
+#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
+#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
+#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
+#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
+#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
+#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
+#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
+
+#define erts_smp_atomic_init_relb erts_atomic_init_relb
+#define erts_smp_atomic_set_relb erts_atomic_set_relb
+#define erts_smp_atomic_read_relb erts_atomic_read_relb
+#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
+#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
+#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
+#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
+#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
+#define erts_smp_atomic_add_relb erts_atomic_add_relb
+#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
+#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
+#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
+#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
+
+#define erts_smp_atomic_init_rb erts_atomic_init_rb
+#define erts_smp_atomic_set_rb erts_atomic_set_rb
+#define erts_smp_atomic_read_rb erts_atomic_read_rb
+#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
+#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
+#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
+#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
+#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
+#define erts_smp_atomic_add_rb erts_atomic_add_rb
+#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
+#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
+#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
+#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
+
+#define erts_smp_atomic_init_wb erts_atomic_init_wb
+#define erts_smp_atomic_set_wb erts_atomic_set_wb
+#define erts_smp_atomic_read_wb erts_atomic_read_wb
+#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
+#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
+#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
+#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
+#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
+#define erts_smp_atomic_add_wb erts_atomic_add_wb
+#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
+#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
+#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
+#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
+#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
+#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
+#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
+#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
+#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
+#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
+#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
+#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
+#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
+#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
+#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
+#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
+
+#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
+#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
+#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
+#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
+#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
+#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
+#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
+#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
+#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
+#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
+#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
+#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
+#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
+
+#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
+#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
+#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
+#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
+#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
+#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
+#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
+#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
+#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
+#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
+#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
+#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
+#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
+
+#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
+#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
+#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
+#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
+#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
+#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
+#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
+#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
+#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
+#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
+#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
+#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
+#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
+
+#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
+#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
+#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
+#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
+#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
+#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
+#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
+#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
+#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
+#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
+#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
+#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
+#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
+
+#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
+#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
+#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
+#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
+#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
+#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
+#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
+#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
+#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
+#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
+#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
+#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
+#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
+
+#else /* !ERTS_SMP */
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_no_atomic_set
+#define erts_smp_atomic_set_nob erts_no_atomic_set
+#define erts_smp_atomic_read_nob erts_no_atomic_read
+#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_nob erts_no_atomic_inc
+#define erts_smp_atomic_dec_nob erts_no_atomic_dec
+#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_smp_atomic_add_nob erts_no_atomic_add
+#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_mb erts_no_atomic_set
+#define erts_smp_atomic_set_mb erts_no_atomic_set
+#define erts_smp_atomic_read_mb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_mb erts_no_atomic_inc
+#define erts_smp_atomic_dec_mb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_smp_atomic_add_mb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_acqb erts_no_atomic_set
+#define erts_smp_atomic_set_acqb erts_no_atomic_set
+#define erts_smp_atomic_read_acqb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
+#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_smp_atomic_add_acqb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_relb erts_no_atomic_set
+#define erts_smp_atomic_set_relb erts_no_atomic_set
+#define erts_smp_atomic_read_relb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_relb erts_no_atomic_inc
+#define erts_smp_atomic_dec_relb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_smp_atomic_add_relb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_rb erts_no_atomic_set
+#define erts_smp_atomic_set_rb erts_no_atomic_set
+#define erts_smp_atomic_read_rb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_rb erts_no_atomic_inc
+#define erts_smp_atomic_dec_rb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_smp_atomic_add_rb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_wb erts_no_atomic_set
+#define erts_smp_atomic_set_wb erts_no_atomic_set
+#define erts_smp_atomic_read_wb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_wb erts_no_atomic_inc
+#define erts_smp_atomic_dec_wb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_smp_atomic_add_wb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_no_atomic32_set
+#define erts_smp_atomic32_set_nob erts_no_atomic32_set
+#define erts_smp_atomic32_read_nob erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_nob erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_mb erts_no_atomic32_set
+#define erts_smp_atomic32_set_mb erts_no_atomic32_set
+#define erts_smp_atomic32_read_mb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_mb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_relb erts_no_atomic32_set
+#define erts_smp_atomic32_set_relb erts_no_atomic32_set
+#define erts_smp_atomic32_read_relb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_relb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_rb erts_no_atomic32_set
+#define erts_smp_atomic32_set_rb erts_no_atomic32_set
+#define erts_smp_atomic32_read_rb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_rb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_wb erts_no_atomic32_set
+#define erts_smp_atomic32_set_wb erts_no_atomic32_set
+#define erts_smp_atomic32_read_wb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_wb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !ERTS_SMP */
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -655,434 +1004,6 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
}
ERTS_GLB_INLINE void
-erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_init(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_set(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_read(erts_smp_atomic_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic_read(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_inctest(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_inctest(incp);
-#else
- return ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_dectest(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_dectest(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_inc(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
- erts_atomic_inc(incp);
-#else
- ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_dec(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic_dec(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- return erts_atomic_addtest(addp, i);
-#else
- return *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_add(addp, i);
-#else
- *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new)
-{
-#ifdef ERTS_SMP
- return erts_atomic_xchg(xchgp, new);
-#else
- erts_aint_t old;
- old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg(xchgp, new, expected);
-#else
- erts_aint_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic_bor(var, mask);
-#else
- erts_aint_t old;
- old = *var;
- *var |= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic_band(var, mask);
-#else
- erts_aint_t old;
- old = *var;
- *var &= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_read_acqb(erts_smp_atomic_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_dectest_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg_acqb(xchgp, new, exp);
-#else
- erts_aint_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg_relb(xchgp, new, exp);
-#else
- erts_aint_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_init(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_set(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read(erts_smp_atomic32_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_read(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_inctest(incp);
-#else
- return ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_dectest(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_inc(erts_smp_atomic32_t *incp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_inc(incp);
-#else
- ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_dec(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_addtest(addp, i);
-#else
- return *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_add(addp, i);
-#else
- *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_xchg(xchgp, new);
-#else
- erts_aint32_t old;
- old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg(xchgp, new, expected);
-#else
- erts_aint32_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_bor(var, mask);
-#else
- erts_aint32_t old;
- old = *var;
- *var |= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_band(var, mask);
-#else
- erts_aint32_t old;
- old = *var;
- *var &= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_dectest_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg_acqb(xchgp, new, exp);
-#else
- erts_aint32_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg_relb(xchgp, new, exp);
-#else
- erts_aint32_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
@@ -1308,3 +1229,37 @@ erts_smp_thr_sigwait(const sigset_t *set, int *sig)
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
#endif /* ERL_SMP_H */
+
+#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#undef erts_smp_atomic_init
+#undef erts_smp_atomic_set
+#undef erts_smp_atomic_read
+#undef erts_smp_atomic_inctest
+#undef erts_smp_atomic_dectest
+#undef erts_smp_atomic_inc
+#undef erts_smp_atomic_dec
+#undef erts_smp_atomic_addtest
+#undef erts_smp_atomic_add
+#undef erts_smp_atomic_xchg
+#undef erts_smp_atomic_cmpxchg
+#undef erts_smp_atomic_bor
+#undef erts_smp_atomic_band
+
+#undef erts_smp_atomic32_init
+#undef erts_smp_atomic32_set
+#undef erts_smp_atomic32_read
+#undef erts_smp_atomic32_inctest
+#undef erts_smp_atomic32_dectest
+#undef erts_smp_atomic32_inc
+#undef erts_smp_atomic32_dec
+#undef erts_smp_atomic32_addtest
+#undef erts_smp_atomic32_add
+#undef erts_smp_atomic32_xchg
+#undef erts_smp_atomic32_cmpxchg
+#undef erts_smp_atomic32_bor
+#undef erts_smp_atomic32_band
+
+#endif
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 8c9cace0c5..9b897ffd24 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -28,6 +28,11 @@
#define ERTS_SPIN_BODY ETHR_SPIN_BODY
#include "sys.h"
+
+typedef struct { SWord sint[2]; } erts_no_dw_atomic_t;
+typedef SWord erts_no_atomic_t;
+typedef Sint32 erts_no_atomic32_t;
+
#ifdef USE_THREADS
#define ETHR_TRY_INLINE_FUNCS
@@ -99,10 +104,12 @@ typedef ethr_rwmutex_opt erts_rwmtx_opt_t;
typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
-typedef ethr_sint_t erts_aint_t;
-typedef ethr_atomic_t erts_atomic_t;
-typedef ethr_sint32_t erts_aint32_t;
-typedef ethr_atomic32_t erts_atomic32_t;
+#define erts_dw_aint_t ethr_dw_sint_t
+#define erts_dw_atomic_t ethr_dw_atomic_t
+#define erts_aint_t ethr_sint_t
+#define erts_atomic_t ethr_atomic_t
+#define erts_aint32_t ethr_sint32_t
+#define erts_atomic32_t ethr_atomic32_t
/* spinlock */
typedef struct {
@@ -164,10 +171,12 @@ typedef struct {
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;
-typedef SWord erts_aint_t;
-typedef SWord erts_atomic_t;
-typedef SWord erts_aint32_t;
-typedef SWord erts_atomic32_t;
+#define erts_dw_aint_t erts_no_dw_atomic_t
+#define erts_dw_atomic_t erts_no_dw_atomic_t
+#define erts_aint_t SWord
+#define erts_atomic_t erts_no_atomic_t
+#define erts_aint32_t Sint32
+#define erts_atomic32_t erts_no_atomic32_t
#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
@@ -247,65 +256,51 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp);
-ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp);
-ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp,
- erts_aint_t new);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var);
-ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp);
-ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp);
-ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp,
- erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp,
- erts_aint32_t new);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var,
- erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var,
- erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var);
-ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var,
- erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
+
+ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+ erts_no_dw_atomic_t *val,
+ erts_no_dw_atomic_t *old_val);
+ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
+ erts_aint_t i);
+ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
+ erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
+ erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
+ erts_aint_t new);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
+ erts_aint_t new,
+ erts_aint_t expected);
+ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
+ erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
+ erts_aint32_t i);
+ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
+ erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
+ erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
+ erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
+ erts_aint32_t new);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
+ erts_aint32_t new,
+ erts_aint32_t expected);
+
ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
char *name,
Eterm extra,
@@ -362,6 +357,430 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
+/*
+ * Functions implementing atomic operations with with no (nob),
+ * full (mb), acquire (acqb), release (relb), read (rb), and
+ * write (wb) memory barriers.
+ *
+ * If thread support has been disabled, they are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply memory barriers.
+ */
+
+#ifdef USE_THREADS
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob ethr_dw_atomic_init
+#define erts_dw_atomic_set_nob ethr_dw_atomic_set
+#define erts_dw_atomic_read_nob ethr_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
+#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
+#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
+#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb
+
+#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
+#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
+#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
+#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb
+
+#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
+#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
+#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
+#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
+
+#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
+#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
+#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
+#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb
+
+#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
+#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
+#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
+#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob ethr_atomic_init
+#define erts_atomic_set_nob ethr_atomic_set
+#define erts_atomic_read_nob ethr_atomic_read
+#define erts_atomic_inc_read_nob ethr_atomic_inc_read
+#define erts_atomic_dec_read_nob ethr_atomic_dec_read
+#define erts_atomic_inc_nob ethr_atomic_inc
+#define erts_atomic_dec_nob ethr_atomic_dec
+#define erts_atomic_add_read_nob ethr_atomic_add_read
+#define erts_atomic_add_nob ethr_atomic_add
+#define erts_atomic_read_bor_nob ethr_atomic_read_bor
+#define erts_atomic_read_band_nob ethr_atomic_read_band
+#define erts_atomic_xchg_nob ethr_atomic_xchg
+#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg
+
+#define erts_atomic_init_mb ethr_atomic_init_mb
+#define erts_atomic_set_mb ethr_atomic_set_mb
+#define erts_atomic_read_mb ethr_atomic_read_mb
+#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
+#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
+#define erts_atomic_inc_mb ethr_atomic_inc_mb
+#define erts_atomic_dec_mb ethr_atomic_dec_mb
+#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
+#define erts_atomic_add_mb ethr_atomic_add_mb
+#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
+#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
+#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
+#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb
+
+#define erts_atomic_init_acqb ethr_atomic_init_acqb
+#define erts_atomic_set_acqb ethr_atomic_set_acqb
+#define erts_atomic_read_acqb ethr_atomic_read_acqb
+#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
+#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
+#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
+#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
+#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
+#define erts_atomic_add_acqb ethr_atomic_add_acqb
+#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
+#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
+#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
+#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+
+#define erts_atomic_init_relb ethr_atomic_init_relb
+#define erts_atomic_set_relb ethr_atomic_set_relb
+#define erts_atomic_read_relb ethr_atomic_read_relb
+#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
+#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
+#define erts_atomic_inc_relb ethr_atomic_inc_relb
+#define erts_atomic_dec_relb ethr_atomic_dec_relb
+#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
+#define erts_atomic_add_relb ethr_atomic_add_relb
+#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
+#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
+#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
+#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+
+#define erts_atomic_init_rb ethr_atomic_init_rb
+#define erts_atomic_set_rb ethr_atomic_set_rb
+#define erts_atomic_read_rb ethr_atomic_read_rb
+#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
+#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
+#define erts_atomic_inc_rb ethr_atomic_inc_rb
+#define erts_atomic_dec_rb ethr_atomic_dec_rb
+#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
+#define erts_atomic_add_rb ethr_atomic_add_rb
+#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
+#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
+#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
+#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb
+
+#define erts_atomic_init_wb ethr_atomic_init_wb
+#define erts_atomic_set_wb ethr_atomic_set_wb
+#define erts_atomic_read_wb ethr_atomic_read_wb
+#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
+#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
+#define erts_atomic_inc_wb ethr_atomic_inc_wb
+#define erts_atomic_dec_wb ethr_atomic_dec_wb
+#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
+#define erts_atomic_add_wb ethr_atomic_add_wb
+#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
+#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
+#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
+#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob ethr_atomic32_init
+#define erts_atomic32_set_nob ethr_atomic32_set
+#define erts_atomic32_read_nob ethr_atomic32_read
+#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
+#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
+#define erts_atomic32_inc_nob ethr_atomic32_inc
+#define erts_atomic32_dec_nob ethr_atomic32_dec
+#define erts_atomic32_add_read_nob ethr_atomic32_add_read
+#define erts_atomic32_add_nob ethr_atomic32_add
+#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
+#define erts_atomic32_read_band_nob ethr_atomic32_read_band
+#define erts_atomic32_xchg_nob ethr_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb ethr_atomic32_init_mb
+#define erts_atomic32_set_mb ethr_atomic32_set_mb
+#define erts_atomic32_read_mb ethr_atomic32_read_mb
+#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
+#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
+#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
+#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
+#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
+#define erts_atomic32_add_mb ethr_atomic32_add_mb
+#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
+#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
+#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
+#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb
+
+#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
+#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
+#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
+#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
+#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
+#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
+#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
+#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
+#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
+#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
+#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
+#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
+#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb
+
+#define erts_atomic32_init_relb ethr_atomic32_init_relb
+#define erts_atomic32_set_relb ethr_atomic32_set_relb
+#define erts_atomic32_read_relb ethr_atomic32_read_relb
+#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
+#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
+#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
+#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
+#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
+#define erts_atomic32_add_relb ethr_atomic32_add_relb
+#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
+#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
+#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
+#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+
+#define erts_atomic32_init_rb ethr_atomic32_init_rb
+#define erts_atomic32_set_rb ethr_atomic32_set_rb
+#define erts_atomic32_read_rb ethr_atomic32_read_rb
+#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
+#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
+#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
+#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
+#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
+#define erts_atomic32_add_rb ethr_atomic32_add_rb
+#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
+#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
+#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
+#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb
+
+#define erts_atomic32_init_wb ethr_atomic32_init_wb
+#define erts_atomic32_set_wb ethr_atomic32_set_wb
+#define erts_atomic32_read_wb ethr_atomic32_read_wb
+#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
+#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
+#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
+#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
+#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
+#define erts_atomic32_add_wb ethr_atomic32_add_wb
+#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
+#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
+#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
+#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
+
+#else /* !USE_THREADS */
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob erts_no_atomic_set
+#define erts_atomic_set_nob erts_no_atomic_set
+#define erts_atomic_read_nob erts_no_atomic_read
+#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_atomic_inc_nob erts_no_atomic_inc
+#define erts_atomic_dec_nob erts_no_atomic_dec
+#define erts_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_atomic_add_nob erts_no_atomic_add
+#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_mb erts_no_atomic_set
+#define erts_atomic_set_mb erts_no_atomic_set
+#define erts_atomic_read_mb erts_no_atomic_read
+#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_atomic_inc_mb erts_no_atomic_inc
+#define erts_atomic_dec_mb erts_no_atomic_dec
+#define erts_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_atomic_add_mb erts_no_atomic_add
+#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_acqb erts_no_atomic_set
+#define erts_atomic_set_acqb erts_no_atomic_set
+#define erts_atomic_read_acqb erts_no_atomic_read
+#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_atomic_inc_acqb erts_no_atomic_inc
+#define erts_atomic_dec_acqb erts_no_atomic_dec
+#define erts_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_atomic_add_acqb erts_no_atomic_add
+#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_relb erts_no_atomic_set
+#define erts_atomic_set_relb erts_no_atomic_set
+#define erts_atomic_read_relb erts_no_atomic_read
+#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_atomic_inc_relb erts_no_atomic_inc
+#define erts_atomic_dec_relb erts_no_atomic_dec
+#define erts_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_atomic_add_relb erts_no_atomic_add
+#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_rb erts_no_atomic_set
+#define erts_atomic_set_rb erts_no_atomic_set
+#define erts_atomic_read_rb erts_no_atomic_read
+#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_atomic_inc_rb erts_no_atomic_inc
+#define erts_atomic_dec_rb erts_no_atomic_dec
+#define erts_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_atomic_add_rb erts_no_atomic_add
+#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_wb erts_no_atomic_set
+#define erts_atomic_set_wb erts_no_atomic_set
+#define erts_atomic_read_wb erts_no_atomic_read
+#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_atomic_inc_wb erts_no_atomic_inc
+#define erts_atomic_dec_wb erts_no_atomic_dec
+#define erts_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_atomic_add_wb erts_no_atomic_add
+#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob erts_no_atomic32_set
+#define erts_atomic32_set_nob erts_no_atomic32_set
+#define erts_atomic32_read_nob erts_no_atomic32_read
+#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_atomic32_add_nob erts_no_atomic32_add
+#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb erts_no_atomic32_set
+#define erts_atomic32_set_mb erts_no_atomic32_set
+#define erts_atomic32_read_mb erts_no_atomic32_read
+#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_atomic32_add_mb erts_no_atomic32_add
+#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_acqb erts_no_atomic32_set
+#define erts_atomic32_set_acqb erts_no_atomic32_set
+#define erts_atomic32_read_acqb erts_no_atomic32_read
+#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_atomic32_add_acqb erts_no_atomic32_add
+#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_relb erts_no_atomic32_set
+#define erts_atomic32_set_relb erts_no_atomic32_set
+#define erts_atomic32_read_relb erts_no_atomic32_read
+#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_atomic32_add_relb erts_no_atomic32_add
+#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_rb erts_no_atomic32_set
+#define erts_atomic32_set_rb erts_no_atomic32_set
+#define erts_atomic32_read_rb erts_no_atomic32_read
+#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_atomic32_add_rb erts_no_atomic32_add
+#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_wb erts_no_atomic32_set
+#define erts_atomic32_set_wb erts_no_atomic32_set
+#define erts_atomic32_read_wb erts_no_atomic32_read
+#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_atomic32_add_wb erts_no_atomic32_add
+#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !USE_THREADS */
+
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
@@ -995,428 +1414,206 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
#endif
}
+/* No atomic ops */
+
ERTS_GLB_INLINE void
-erts_atomic_init(erts_atomic_t *var, erts_aint_t i)
+erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
-#ifdef USE_THREADS
- ethr_atomic_init(var, i);
-#else
- *var = i;
-#endif
+ var->sint[0] = val->sint[0];
+ var->sint[1] = val->sint[1];
}
ERTS_GLB_INLINE void
-erts_atomic_set(erts_atomic_t *var, erts_aint_t i)
+erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
+{
+ val->sint[0] = var->sint[0];
+ val->sint[1] = var->sint[1];
+}
+
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+ erts_no_dw_atomic_t *new_val,
+ erts_no_dw_atomic_t *old_val)
+{
+ if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
+ erts_no_dw_atomic_read(var, old_val);
+ return 0;
+ }
+ else {
+ erts_no_dw_atomic_set(var, new_val);
+ return !0;
+ }
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
{
-#ifdef USE_THREADS
- ethr_atomic_set(var, i);
-#else
*var = i;
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_read(erts_atomic_t *var)
+erts_no_atomic_read(erts_no_atomic_t *var)
{
-#ifdef USE_THREADS
- return ethr_atomic_read(var);
-#else
return *var;
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_inctest(erts_atomic_t *incp)
+erts_no_atomic_inc_read(erts_no_atomic_t *incp)
{
-#ifdef USE_THREADS
- return ethr_atomic_inc_read(incp);
-#else
return ++(*incp);
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_dectest(erts_atomic_t *decp)
+erts_no_atomic_dec_read(erts_no_atomic_t *decp)
{
-#ifdef USE_THREADS
- return ethr_atomic_dec_read(decp);
-#else
return --(*decp);
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic_inc(erts_atomic_t *incp)
+erts_no_atomic_inc(erts_no_atomic_t *incp)
{
-#ifdef USE_THREADS
- ethr_atomic_inc(incp);
-#else
++(*incp);
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic_dec(erts_atomic_t *decp)
+erts_no_atomic_dec(erts_no_atomic_t *decp)
{
-#ifdef USE_THREADS
- ethr_atomic_dec(decp);
-#else
--(*decp);
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i)
+erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
{
-#ifdef USE_THREADS
- return ethr_atomic_add_read(addp, i);
-#else
return *addp += i;
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic_add(erts_atomic_t *addp, erts_aint_t i)
+erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
{
-#ifdef USE_THREADS
- ethr_atomic_add(addp, i);
-#else
*addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new)
-{
-#ifdef USE_THREADS
- return ethr_atomic_xchg(xchgp, new);
-#else
- erts_aint_t old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected)
-{
-#ifdef USE_THREADS
- return ethr_atomic_cmpxchg(xchgp, new, expected);
-#else
- erts_aint_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask)
+erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
{
-#ifdef USE_THREADS
- return ethr_atomic_read_bor(var, mask);
-#else
erts_aint_t old;
old = *var;
*var |= mask;
return old;
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_band(erts_atomic_t *var, erts_aint_t mask)
+erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
{
-#ifdef USE_THREADS
- return ethr_atomic_read_band(var, mask);
-#else
erts_aint_t old;
old = *var;
*var &= mask;
return old;
-#endif
}
ERTS_GLB_INLINE erts_aint_t
-erts_atomic_read_acqb(erts_atomic_t *var)
+erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
{
-#ifdef USE_THREADS
- return ethr_atomic_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i)
-{
-#ifdef USE_THREADS
- ethr_atomic_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_atomic_dec_relb(erts_atomic_t *decp)
-{
-#ifdef USE_THREADS
- ethr_atomic_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_atomic_dectest_relb(erts_atomic_t *decp)
-{
-#ifdef USE_THREADS
- return ethr_atomic_dec_read_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
-{
-#ifdef USE_THREADS
- return ethr_atomic_cmpxchg_acqb(xchgp, new, exp);
-#else
erts_aint_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
+ *xchgp = new;
return old;
-#endif
}
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
+ erts_aint_t new,
+ erts_aint_t expected)
{
-#ifdef USE_THREADS
- return ethr_atomic_cmpxchg_relb(xchgp, new, exp);
-#else
erts_aint_t old = *xchgp;
- if (old == exp)
+ if (old == expected)
*xchgp = new;
return old;
-#endif
}
/* atomic32 */
ERTS_GLB_INLINE void
-erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef USE_THREADS
- ethr_atomic32_init(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i)
+erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
{
-#ifdef USE_THREADS
- ethr_atomic32_set(var, i);
-#else
*var = i;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_read(erts_atomic32_t *var)
+erts_no_atomic32_read(erts_no_atomic32_t *var)
{
-#ifdef USE_THREADS
- return ethr_atomic32_read(var);
-#else
return *var;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_inctest(erts_atomic32_t *incp)
+erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
{
-#ifdef USE_THREADS
- return ethr_atomic32_inc_read(incp);
-#else
return ++(*incp);
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_dectest(erts_atomic32_t *decp)
+erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
{
-#ifdef USE_THREADS
- return ethr_atomic32_dec_read(decp);
-#else
return --(*decp);
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic32_inc(erts_atomic32_t *incp)
+erts_no_atomic32_inc(erts_no_atomic32_t *incp)
{
-#ifdef USE_THREADS
- ethr_atomic32_inc(incp);
-#else
++(*incp);
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic32_dec(erts_atomic32_t *decp)
+erts_no_atomic32_dec(erts_no_atomic32_t *decp)
{
-#ifdef USE_THREADS
- ethr_atomic32_dec(decp);
-#else
--(*decp);
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i)
+erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
{
-#ifdef USE_THREADS
- return ethr_atomic32_add_read(addp, i);
-#else
return *addp += i;
-#endif
}
ERTS_GLB_INLINE void
-erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i)
+erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
{
-#ifdef USE_THREADS
- ethr_atomic32_add(addp, i);
-#else
*addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new)
-{
-#ifdef USE_THREADS
- return ethr_atomic32_xchg(xchgp, new);
-#else
- erts_aint32_t old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_cmpxchg(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected)
-{
-#ifdef USE_THREADS
- return ethr_atomic32_cmpxchg(xchgp, new, expected);
-#else
- erts_aint32_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask)
+erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
{
-#ifdef USE_THREADS
- return ethr_atomic32_read_bor(var, mask);
-#else
erts_aint32_t old;
old = *var;
*var |= mask;
return old;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask)
+erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
{
-#ifdef USE_THREADS
- return ethr_atomic32_read_band(var, mask);
-#else
erts_aint32_t old;
old = *var;
*var &= mask;
return old;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_read_acqb(erts_atomic32_t *var)
-{
-#ifdef USE_THREADS
- return ethr_atomic32_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef USE_THREADS
- ethr_atomic32_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_atomic32_dec_relb(erts_atomic32_t *decp)
+erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
{
-#ifdef USE_THREADS
- ethr_atomic32_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_dectest_relb(erts_atomic32_t *decp)
-{
-#ifdef USE_THREADS
- return ethr_atomic32_dec_read_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
-{
-#ifdef USE_THREADS
- return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp);
-#else
erts_aint32_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
+ *xchgp = new;
return old;
-#endif
}
ERTS_GLB_INLINE erts_aint32_t
-erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
+erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
+ erts_aint32_t new,
+ erts_aint32_t expected)
{
-#ifdef USE_THREADS
- return ethr_atomic32_cmpxchg_relb(xchgp, new, exp);
-#else
erts_aint32_t old = *xchgp;
- if (old == exp)
+ if (old == expected)
*xchgp = new;
return old;
-#endif
}
/* spinlock */
@@ -1887,3 +2084,37 @@ erts_thr_sigwait(const sigset_t *set, int *sig)
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
#endif /* #ifndef ERL_THREAD_H__ */
+
+#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#undef erts_atomic_init
+#undef erts_atomic_set
+#undef erts_atomic_read
+#undef erts_atomic_inctest
+#undef erts_atomic_dectest
+#undef erts_atomic_inc
+#undef erts_atomic_dec
+#undef erts_atomic_addtest
+#undef erts_atomic_add
+#undef erts_atomic_xchg
+#undef erts_atomic_cmpxchg
+#undef erts_atomic_bor
+#undef erts_atomic_band
+
+#undef erts_atomic32_init
+#undef erts_atomic32_set
+#undef erts_atomic32_read
+#undef erts_atomic32_inctest
+#undef erts_atomic32_dectest
+#undef erts_atomic32_inc
+#undef erts_atomic32_dec
+#undef erts_atomic32_addtest
+#undef erts_atomic32_add
+#undef erts_atomic32_xchg
+#undef erts_atomic32_cmpxchg
+#undef erts_atomic32_bor
+#undef erts_atomic32_band
+
+#endif
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index d0ad73cd81..7a09d30ff6 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -85,8 +85,8 @@ ERTS_GLB_INLINE void erts_do_time_add(long);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); }
-ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); }
+ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg_acqb(&do_time, 0L); }
+ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add_relb(&do_time, elapsed); }
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 499bdd77ba..249df54015 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -200,10 +200,10 @@ erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
ErtsRunQueue *rq1, *rq2;
- rq1 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue);
+ rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
while (1) {
erts_smp_runq_lock(rq1);
- rq2 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue);
+ rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
if (rq1 == rq2)
return rq1;
erts_smp_runq_unlock(rq1);
@@ -542,10 +542,11 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt)
ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck));
if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) {
/* Dead ports are added from the end of the snapshot buffer */
- Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr,
- -(erts_aint_t)sizeof(Eterm));
+ Eterm* tombstone;
+ tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr,
+ -(erts_aint_t)sizeof(Eterm));
ASSERT(tombstone+1 != NULL);
- ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1);
+ ASSERT(prt->snapshot == erts_smp_atomic_read_nob(&erts_ports_snapshot) - 1);
*tombstone = prt->id;
}
/*else no ongoing snapshot or port was already included or created after snapshot */
@@ -1200,11 +1201,11 @@ erts_smp_port_trylock(Port *prt)
#ifdef ERTS_SMP
int res;
- ASSERT(erts_smp_atomic_read(&prt->refc) > 0);
- erts_smp_atomic_inc(&prt->refc);
+ ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
+ erts_smp_atomic_inc_nob(&prt->refc);
res = erts_smp_mtx_trylock(prt->lock);
if (res == EBUSY) {
- erts_smp_atomic_dec(&prt->refc);
+ erts_smp_atomic_dec_nob(&prt->refc);
}
return res;
@@ -1217,8 +1218,8 @@ ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
- ASSERT(erts_smp_atomic_read(&prt->refc) > 0);
- erts_smp_atomic_inc(&prt->refc);
+ ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
+ erts_smp_atomic_inc_nob(&prt->refc);
erts_smp_mtx_lock(prt->lock);
#endif
}
@@ -1229,7 +1230,7 @@ erts_smp_port_unlock(Port *prt)
#ifdef ERTS_SMP
erts_aint_t refc;
erts_smp_mtx_unlock(prt->lock);
- refc = erts_smp_atomic_dectest(&prt->refc);
+ refc = erts_smp_atomic_dec_read_nob(&prt->refc);
ASSERT(refc >= 0);
if (refc == 0)
erts_port_cleanup(prt);
@@ -1298,7 +1299,7 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
}
#ifdef ERTS_SMP
else {
- erts_smp_atomic_inc(&prt->refc);
+ erts_smp_atomic_inc_nob(&prt->refc);
erts_smp_port_state_unlock(prt);
if (no_proc_locks)
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index df5f8b22a3..151c776a3d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -244,8 +244,8 @@ get_free_port(void)
}
port->status = ERTS_PORT_SFLG_INITIALIZING;
#ifdef ERTS_SMP
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 0);
- erts_smp_atomic_set(&port->refc, 2); /* Port alive + lock */
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
+ erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */
#endif
erts_smp_port_state_unlock(port);
return num & port_num_mask;
@@ -327,7 +327,7 @@ port_cleanup(Port *prt)
#ifdef ERTS_SMP
ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&prt->refc) == 0);
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK);
@@ -425,11 +425,11 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
erts_smp_runq_lock(runq);
erts_smp_port_state_lock(prt);
prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
- prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot);
+ prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot);
old_name = prt->name;
prt->name = new_name;
#ifdef ERTS_SMP
- erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq);
+ erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
#endif
ASSERT(!prt->drv_ptr);
prt->drv_ptr = driver;
@@ -590,8 +590,8 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */
erts_smp_port_state_lock(port);
port->status = ERTS_PORT_SFLG_FREE;
#ifdef ERTS_SMP
- ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 2);
- erts_smp_atomic_set(&port->refc, 0);
+ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2);
+ erts_smp_atomic_set_nob(&port->refc, 0);
#endif
erts_smp_port_state_unlock(port);
return -3;
@@ -1206,7 +1206,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
}
}
p->bytes_out += size;
- erts_smp_atomic_add(&erts_bytes_out, size);
+ erts_smp_atomic_add_nob(&erts_bytes_out, size);
#ifdef ERTS_SMP
if (p->xports)
@@ -1277,13 +1277,13 @@ void init_io(void)
erts_port = (Port *) erts_alloc(ERTS_ALC_T_PORT_TABLE,
erts_max_ports * sizeof(Port));
- erts_smp_atomic_init(&erts_bytes_out, 0);
- erts_smp_atomic_init(&erts_bytes_in, 0);
+ erts_smp_atomic_init_nob(&erts_bytes_out, 0);
+ erts_smp_atomic_init_nob(&erts_bytes_in, 0);
for (i = 0; i < erts_max_ports; i++) {
erts_port_task_init_sched(&erts_port[i].sched);
#ifdef ERTS_SMP
- erts_smp_atomic_init(&erts_port[i].refc, 0);
+ erts_smp_atomic_init_nob(&erts_port[i].refc, 0);
erts_port[i].lock = NULL;
erts_port[i].xports = NULL;
erts_smp_spinlock_init_x(&erts_port[i].state_lck, "port_state", make_small(i));
@@ -1300,7 +1300,7 @@ void init_io(void)
erts_port[i].port_data_lock = NULL;
}
- erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0);
+ erts_smp_atomic32_init_nob(&erts_ports_snapshot, (erts_aint32_t) 0);
last_port_num = 0;
erts_smp_spinlock_init(&get_free_port_lck, "get_free_port");
@@ -3253,7 +3253,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen,
return 0;
prt->bytes_in += (hlen + len);
- erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len));
+ erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
return erts_net_message(prt,
prt->dist_entry,
@@ -3288,7 +3288,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len)
return 0;
prt->bytes_in += (hlen + len);
- erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len));
+ erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
if (len == 0)
return erts_net_message(prt,
@@ -3365,7 +3365,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip)
/* XXX handle distribution !!! */
prt->bytes_in += (hlen + size);
- erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size));
+ erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size));
deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size);
return 0;
}
@@ -3539,13 +3539,13 @@ pdl_init(void)
static ERTS_INLINE void
pdl_init_refc(ErlDrvPDL pdl)
{
- erts_atomic_init(&pdl->refc, 1);
+ erts_atomic_init_nob(&pdl->refc, 1);
}
static ERTS_INLINE ErlDrvSInt
pdl_read_refc(ErlDrvPDL pdl)
{
- erts_aint_t refc = erts_atomic_read(&pdl->refc);
+ erts_aint_t refc = erts_atomic_read_nob(&pdl->refc);
ERTS_LC_ASSERT(refc >= 0);
return (ErlDrvSInt) refc;
}
@@ -3553,14 +3553,14 @@ pdl_read_refc(ErlDrvPDL pdl)
static ERTS_INLINE void
pdl_inc_refc(ErlDrvPDL pdl)
{
- erts_atomic_inc(&pdl->refc);
+ erts_atomic_inc_nob(&pdl->refc);
ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1);
}
static ERTS_INLINE ErlDrvSInt
pdl_inctest_refc(ErlDrvPDL pdl)
{
- erts_aint_t refc = erts_atomic_inctest(&pdl->refc);
+ erts_aint_t refc = erts_atomic_inc_read_nob(&pdl->refc);
ERTS_LC_ASSERT(refc > 1);
return (ErlDrvSInt) refc;
}
@@ -3569,7 +3569,7 @@ pdl_inctest_refc(ErlDrvPDL pdl)
static ERTS_INLINE void
pdl_dec_refc(ErlDrvPDL pdl)
{
- erts_atomic_dec(&pdl->refc);
+ erts_atomic_dec_nob(&pdl->refc);
ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 0);
}
#endif
@@ -3577,7 +3577,7 @@ pdl_dec_refc(ErlDrvPDL pdl)
static ERTS_INLINE ErlDrvSInt
pdl_dectest_refc(ErlDrvPDL pdl)
{
- erts_aint_t refc = erts_atomic_dectest(&pdl->refc);
+ erts_aint_t refc = erts_atomic_dec_read_nob(&pdl->refc);
ERTS_LC_ASSERT(refc >= 0);
return (ErlDrvSInt) refc;
}
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 4c54e19cdb..3326e5cc2a 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -61,7 +61,7 @@ static ERTS_INLINE int align_up_pow2(int val)
*/
static void rehash(SafeHash* h, int grow_limit)
{
- if (erts_smp_atomic_xchg(&h->is_rehashing, 1) != 0) {
+ if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) {
return; /* already in progress */
}
if (h->grow_limit == grow_limit) {
@@ -166,8 +166,8 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size,
h->name = name;
h->fun = fun;
set_size(h,size);
- erts_smp_atomic_init(&h->is_rehashing, 0);
- erts_smp_atomic_init(&h->nitems, 0);
+ erts_smp_atomic_init_nob(&h->is_rehashing, 0);
+ erts_smp_atomic_init_nob(&h->nitems, 0);
for (i=0; i<SAFE_HASH_LOCK_CNT; i++) {
erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash");
}
@@ -222,7 +222,7 @@ void* safe_hash_put(SafeHash* h, void* tmpl)
*head = b;
grow_limit = h->grow_limit;
erts_smp_mtx_unlock(lock);
- if (erts_smp_atomic_inctest(&h->nitems) > grow_limit) {
+ if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) {
rehash(h, grow_limit);
}
return (void*) b;
@@ -245,7 +245,7 @@ void* safe_hash_erase(SafeHash* h, void* tmpl)
if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) {
*prevp = b->next;
erts_smp_mtx_unlock(lock);
- erts_smp_atomic_dec(&h->nitems);
+ erts_smp_atomic_dec_nob(&h->nitems);
h->fun.free((void*)b);
return tmpl;
}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index e64c43de6e..669a601b35 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -340,7 +340,8 @@ int erts_send_warning_to_logger_str_nogl(char *);
#ifdef ERTS_WANT_BREAK_HANDLING
# ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
-# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested))
+# define ERTS_BREAK_REQUESTED \
+ ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
# else
extern volatile int erts_break_requested;
# define ERTS_BREAK_REQUESTED erts_break_requested
@@ -354,7 +355,7 @@ void erts_do_break_handling(void);
# else
# ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
-# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1))
+# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
# else
extern volatile int erts_got_sigusr1;
# define ERTS_GOT_SIGUSR1 erts_got_sigusr1
@@ -363,11 +364,15 @@ extern volatile int erts_got_sigusr1;
#endif
#ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_writing_erl_crash_dump;
+extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
+extern erts_tsd_key_t erts_is_crash_dumping_key;
+#define ERTS_SOMEONE_IS_CRASH_DUMPING \
+ ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
- ((int) erts_smp_atomic_read(&erts_writing_erl_crash_dump))
+ ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
extern volatile int erts_writing_erl_crash_dump;
+#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
@@ -877,7 +882,7 @@ ERTS_GLB_INLINE int
erts_smp_pending_system_block(void)
{
#ifdef ERTS_SMP
- return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block);
+ return (int) erts_smp_atomic32_read_nob(&erts_system_block_state.do_block);
#else
return 0;
#endif
@@ -913,7 +918,7 @@ erts_smp_set_activity(erts_activity_t old_activity,
case ERTS_ACTIVITY_UNDEFINED:
break;
case ERTS_ACTIVITY_WAIT:
- erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait);
+ erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.wait);
if (locked) {
/* You are not allowed to leave activity waiting
* without supplying the possibility to block
@@ -924,10 +929,10 @@ erts_smp_set_activity(erts_activity_t old_activity,
}
break;
case ERTS_ACTIVITY_GC:
- erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc);
+ erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.gc);
break;
case ERTS_ACTIVITY_IO:
- erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io);
+ erts_smp_atomic32_dec_acqb(&erts_system_block_state.in_activity.io);
break;
default:
erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY,
@@ -943,13 +948,13 @@ erts_smp_set_activity(erts_activity_t old_activity,
case ERTS_ACTIVITY_UNDEFINED:
break;
case ERTS_ACTIVITY_WAIT:
- erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait);
+ erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.wait);
break;
case ERTS_ACTIVITY_GC:
- erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc);
+ erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.gc);
break;
case ERTS_ACTIVITY_IO:
- erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io);
+ erts_smp_atomic32_inc_mb(&erts_system_block_state.in_activity.io);
break;
default:
erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY,
@@ -1001,27 +1006,27 @@ ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
- erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val);
+ erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}
ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
- erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+ erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
"erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#else
- erts_smp_atomic_inc((erts_smp_atomic_t *) refcp);
+ erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
}
ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
- erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+ erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
@@ -1035,20 +1040,20 @@ ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
- erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+ erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
"erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
val, min_val);
#else
- erts_smp_atomic_dec((erts_smp_atomic_t *) refcp);
+ erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
}
ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
- erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+ erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
@@ -1062,20 +1067,20 @@ ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
- erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff);
+ erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
"erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
diff, val, min_val);
#else
- erts_smp_atomic_add((erts_smp_atomic_t *) refcp, diff);
+ erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
}
ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
- erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp);
+ erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
if (val < min_val)
erl_exit(ERTS_ABORT_EXIT,
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index a00faff912..8fa8c1cfe0 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -108,9 +108,9 @@ static ErlTimer *tiw_min_ptr;
static int itime; /* Constant after init */
erts_smp_atomic_t do_time; /* set at clock interrupt */
-static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); }
+static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read_acqb(&do_time); }
static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); }
-static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); }
+static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init_nob(&do_time, 0L); }
/* get the time (in units of itime) to the next timeout,
or -1 if there are no timeouts */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index a17de717bc..3f6accba2d 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3689,14 +3689,16 @@ threads_not_under_control(void)
{
erts_aint32_t res = system_block_state.threads_to_block;
+ ERTS_THR_MEMORY_BARRIER;
+
/* Waiting is always an allowed activity... */
- res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait);
+ res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.wait);
if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC)
- res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc);
+ res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.gc);
if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO)
- res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io);
+ res -= erts_smp_atomic32_read_nob(&erts_system_block_state.in_activity.io);
if (res < 0) {
ASSERT(0);
@@ -3756,7 +3758,7 @@ erts_block_system(Uint32 allowed_activities)
}
else {
- erts_smp_atomic32_inc(&erts_system_block_state.do_block);
+ erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block);
/* Someone else might be waiting for us to block... */
if (do_block) {
@@ -3808,11 +3810,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities)
another_blocker = erts_smp_pending_system_block();
system_block_state.emergency = 1;
- erts_smp_atomic32_inc(&erts_system_block_state.do_block);
+ erts_smp_atomic32_inc_nob(&erts_system_block_state.do_block);
if (another_blocker) {
if (is_blocker()) {
- erts_smp_atomic32_dec(&erts_system_block_state.do_block);
+ erts_smp_atomic32_dec_nob(&erts_system_block_state.do_block);
res = 0;
goto done;
}
@@ -3869,7 +3871,7 @@ erts_release_system(void)
if (system_block_state.recursive_block)
system_block_state.recursive_block--;
else {
- do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block);
+ do_block = erts_smp_atomic32_dec_read_nob(&erts_system_block_state.do_block);
system_block_state.have_blocker = 0;
if (is_blockable_thread())
system_block_state.threads_to_block++;
@@ -4004,10 +4006,10 @@ erts_system_block_init(void)
/* Global state... */
- erts_smp_atomic32_init(&erts_system_block_state.do_block, 0);
- erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0);
- erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0);
- erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0);
+ erts_smp_atomic32_init_nob(&erts_system_block_state.do_block, 0);
+ erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.wait, 0);
+ erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.gc, 0);
+ erts_smp_atomic32_init_nob(&erts_system_block_state.in_activity.io, 0);
/* Make sure blockable threads unregister when exiting... */
erts_smp_install_exit_handler(erts_unregister_blockable_thread);
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 8d31348496..dfb4ca794a 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -334,7 +334,7 @@ char *hipe_bs_allocate(int len)
bptr = erts_bin_nrml_alloc(len);
bptr->flags = 0;
bptr->orig_size = len;
- erts_smp_atomic_init(&bptr->refc, 1);
+ erts_smp_atomic_init_nob(&bptr->refc, 1);
return bptr->orig_bytes;
}
@@ -584,7 +584,7 @@ void hipe_clear_timeout(Process *c_p)
void hipe_atomic_inc(int *counter)
{
- erts_smp_atomic_inc((erts_smp_atomic_t*)counter);
+ erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter);
}
#endif
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index cd4de21d65..57321259f9 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -218,7 +218,7 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
#ifdef ERTS_SMP
struct removed_fd *fdlp;
ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
- if (erts_smp_atomic_read(&psi->in_poll_wait)) {
+ if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) {
state->remove_cnt++;
ASSERT(state->remove_cnt > 0);
fdlp = removed_fd_alloc();
@@ -333,7 +333,7 @@ grow_drv_ev_state(int min_ix)
new_len = max_fds;
erts_smp_mtx_lock(&drv_ev_state_grow_lock);
- if (erts_smp_atomic_read(&drv_ev_state_len) <= min_ix) {
+ if (erts_smp_atomic_read_nob(&drv_ev_state_len) <= min_ix) {
for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
erts_smp_mtx_lock(&drv_ev_state_locks[i].lck);
}
@@ -343,7 +343,7 @@ grow_drv_ev_state(int min_ix)
sizeof(ErtsDrvEventState)*new_len)
: erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
sizeof(ErtsDrvEventState)*new_len));
- for (i = erts_smp_atomic_read(&drv_ev_state_len); i < new_len; i++) {
+ for (i = erts_smp_atomic_read_nob(&drv_ev_state_len); i < new_len; i++) {
drv_ev_state[i].fd = (ErtsSysFdType) i;
drv_ev_state[i].driver.select = NULL;
drv_ev_state[i].events = 0;
@@ -351,7 +351,7 @@ grow_drv_ev_state(int min_ix)
drv_ev_state[i].type = ERTS_EV_TYPE_NONE;
drv_ev_state[i].flags = 0;
}
- erts_smp_atomic_set(&drv_ev_state_len, new_len);
+ erts_smp_atomic_set_nob(&drv_ev_state_len, new_len);
for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck);
}
@@ -497,7 +497,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
&& erts_lc_is_port_locked(erts_drvport2port(ix)));
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
- if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) {
+ if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
if (fd < 0) {
return -1;
}
@@ -709,7 +709,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
&& erts_lc_is_port_locked(erts_drvport2port(ix)));
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
- if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) {
+ if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
if (fd < 0)
return -1;
if (fd >= max_fds) {
@@ -1156,7 +1156,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
pollres_len = sizeof(pollres)/sizeof(ErtsPollResFd);
- erts_smp_atomic_set(&pollset.in_poll_wait, 1);
+ erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1);
poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, &wait_time);
@@ -1173,7 +1173,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
#endif
if (poll_ret != 0) {
- erts_smp_atomic_set(&pollset.in_poll_wait, 0);
+ erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
forget_removed(&pollset);
if (poll_ret == EAGAIN) {
goto restart;
@@ -1304,7 +1304,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
#endif
}
- erts_smp_atomic_set(&pollset.in_poll_wait, 0);
+ erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
forget_removed(&pollset);
}
@@ -1419,7 +1419,7 @@ static void drv_ev_state_free(void *des)
void
ERTS_CIO_EXPORT(erts_init_check_io)(void)
{
- erts_smp_atomic_init(&pollset.in_poll_wait, 0);
+ erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0);
ERTS_CIO_POLL_INIT();
pollset.ps = ERTS_CIO_NEW_POLLSET();
@@ -1441,7 +1441,7 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
#endif
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
max_fds = ERTS_CIO_POLL_MAX_FDS();
- erts_smp_atomic_init(&drv_ev_state_len, 0);
+ erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
drv_ev_state = NULL;
erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow");
#else
@@ -1479,7 +1479,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
ERTS_CIO_POLL_INFO(pollset.ps, &pi);
res = pi.memory_size;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
- res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len);
+ res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
#else
res += safe_hash_table_sz(&drv_ev_state_tab);
{
@@ -1506,7 +1506,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
ERTS_CIO_POLL_INFO(pollset.ps, &pi);
memory_size = pi.memory_size;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
- memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len);
+ memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
#else
memory_size += safe_hash_table_sz(&drv_ev_state_tab);
{
@@ -1886,7 +1886,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void)
counters.num_errors = 0;
#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
- len = erts_smp_atomic_read(&drv_ev_state_len);
+ len = erts_smp_atomic_read_nob(&drv_ev_state_len);
for (fd = 0; fd < len; fd++) {
doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters);
}
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index f5c785d683..81f1c95020 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -124,11 +124,11 @@
erts_smp_mtx_unlock(&(PS)->mtx)
#define ERTS_POLLSET_SET_POLLED_CHK(PS) \
- ((int) erts_atomic32_xchg(&(PS)->polled, (erts_aint32_t) 1))
+ ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1))
#define ERTS_POLLSET_UNSET_POLLED(PS) \
- erts_atomic32_set(&(PS)->polled, (erts_aint32_t) 0)
+ erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0)
#define ERTS_POLLSET_IS_POLLED(PS) \
- ((int) erts_atomic32_read(&(PS)->polled))
+ ((int) erts_atomic32_read_nob(&(PS)->polled))
#else
@@ -142,11 +142,11 @@
#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
- erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 1)
+ erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
- erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 0)
+ erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
- ((int) erts_smp_atomic32_read(&(PS)->have_update_requests))
+ ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests))
#else
#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS)
#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS)
@@ -346,7 +346,7 @@ static ERTS_INLINE void
reset_wakeup_state(ErtsPollSet ps)
{
#ifdef ERTS_SMP
- erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+ erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
ERTS_THR_MEMORY_BARRIER;
#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
ps->wakeup_state = 0;
@@ -369,7 +369,7 @@ static ERTS_INLINE int
is_interrupted_reset(ErtsPollSet ps)
{
#ifdef ERTS_SMP
- return (erts_atomic32_xchg(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
+ return (erts_atomic32_xchg_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
== ERTS_POLL_WOKEN_INTR);
#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
int res = ps->wakeup_state == ERTS_POLL_WOKEN_INTR;
@@ -384,12 +384,12 @@ static ERTS_INLINE void
woke_up(ErtsPollSet ps)
{
#ifdef ERTS_SMP
- erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
if (wakeup_state == ERTS_POLL_NOT_WOKEN)
- (void) erts_atomic32_cmpxchg(&ps->wakeup_state,
- ERTS_POLL_WOKEN,
- ERTS_POLL_NOT_WOKEN);
- ASSERT(erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
+ (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+ ERTS_POLL_WOKEN,
+ ERTS_POLL_NOT_WOKEN);
+ ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
if (ps->wakeup_state == ERTS_POLL_NOT_WOKEN)
ps->wakeup_state = ERTS_POLL_WOKEN;
@@ -417,7 +417,7 @@ wake_poller(ErtsPollSet ps, int interrupted)
* We might unnecessarily write to the pipe, however,
* that isn't problematic.
*/
- wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
}
wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
@@ -839,7 +839,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
ASSERT(ps->fds_status[fd].used_events);
ps->fds_status[fd].used_events = 0;
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
update_fallback_pollset(ps, fd);
ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
break;
@@ -889,11 +889,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
if (!events) {
buf[buf_len].events = POLLREMOVE;
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
}
else if (!ps->fds_status[fd].used_events) {
buf[buf_len].events = events;
- erts_smp_atomic_inc(&ps->no_of_user_fds);
+ erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
}
else {
if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
@@ -983,12 +983,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
}
if (used_events) {
if (!events) {
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
}
}
else {
if (events)
- erts_smp_atomic_inc(&ps->no_of_user_fds);
+ erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
}
ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
@@ -1062,7 +1062,7 @@ update_pollset(ErtsPollSet ps, int fd)
epe.data.fd = epe_templ.data.fd;
res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
} while (res != 0 && errno == EINTR);
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
ps->fds_status[fd].used_events = 0;
}
@@ -1070,11 +1070,11 @@ update_pollset(ErtsPollSet ps, int fd)
/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
need a non-NULL event pointer even though it is ignored... */
op = EPOLL_CTL_DEL;
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
}
else if (!ps->fds_status[fd].used_events) {
op = EPOLL_CTL_ADD;
- erts_smp_atomic_inc(&ps->no_of_user_fds);
+ erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
}
else {
op = EPOLL_CTL_MOD;
@@ -1124,7 +1124,7 @@ update_pollset(ErtsPollSet ps, int fd)
/* Fall through ... */
case EPOLL_CTL_ADD: {
ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
#if ERTS_POLL_USE_CONCURRENT_UPDATE
if (!*update_fallback) {
*update_fallback = 1;
@@ -1212,7 +1212,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
#if ERTS_POLL_USE_FALLBACK
ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
#endif
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
last_pix = --ps->no_poll_fds;
if (pix != last_pix) {
/* Move last pix to this pix */
@@ -1239,7 +1239,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
|| fd == ps->kp_fd);
#endif
- erts_smp_atomic_inc(&ps->no_of_user_fds);
+ erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
ps->fds_status[fd].pix = pix = ps->no_poll_fds++;
if (pix >= ps->poll_fds_len)
grow_poll_fds(ps, pix);
@@ -1290,7 +1290,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
if (!ps->fds_status[fd].used_events) {
ASSERT(events);
- erts_smp_atomic_inc(&ps->no_of_user_fds);
+ erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
#if ERTS_POLL_USE_FALLBACK
ps->no_select_fds++;
ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
@@ -1298,7 +1298,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
}
else if (!events) {
ASSERT(ps->fds_status[fd].used_events);
- erts_smp_atomic_dec(&ps->no_of_user_fds);
+ erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
ps->fds_status[fd].events = events;
#if ERTS_POLL_USE_FALLBACK
ps->no_select_fds--;
@@ -1896,7 +1896,7 @@ static ERTS_INLINE int
check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
{
ASSERT(!*ps_locked);
- if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0
+ if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0
&& tv->tv_usec == 0 && tv->tv_sec == 0) {
/* Nothing to poll and zero timeout; done... */
return 0;
@@ -1937,7 +1937,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
* the maximum number of file descriptors in the poll set.
*/
struct dvpoll poll_res;
- int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds);
+ int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
#ifdef ERTS_SMP
nfds++; /* Wakeup pipe */
#endif
@@ -2143,10 +2143,10 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
else {
if (ERTS_POLLSET_IS_POLLED(ps))
- erts_smp_atomic_inc(&ps->no_avoided_wakeups);
- erts_smp_atomic_inc(&ps->no_avoided_interrupts);
+ erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups);
+ erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts);
}
- erts_smp_atomic_inc(&ps->no_interrupt_timed);
+ erts_smp_atomic_inc_nob(&ps->no_interrupt_timed);
#endif
}
#endif
@@ -2208,7 +2208,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
ps->internal_fd_limit = 0;
ps->fds_status = NULL;
ps->fds_status_len = 0;
- erts_smp_atomic_init(&ps->no_of_user_fds, 0);
+ erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0);
#if ERTS_POLL_USE_KERNEL_POLL
ps->kp_fd = -1;
#if ERTS_POLL_USE_EPOLL
@@ -2260,14 +2260,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
ps->update_requests.next = NULL;
ps->update_requests.len = 0;
ps->curr_upd_req_block = &ps->update_requests;
- erts_smp_atomic32_init(&ps->have_update_requests, 0);
+ erts_smp_atomic32_init_nob(&ps->have_update_requests, 0);
#endif
#ifdef ERTS_SMP
- erts_atomic32_init(&ps->polled, 0);
+ erts_atomic32_init_nob(&ps->polled, 0);
erts_smp_mtx_init(&ps->mtx, "pollset");
#endif
#ifdef ERTS_SMP
- erts_atomic32_init(&ps->wakeup_state, (erts_aint32_t) 0);
+ erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
ps->wakeup_state = 0;
#endif
@@ -2291,11 +2291,11 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
ps->internal_fd_limit = kp_fd + 1;
ps->kp_fd = kp_fd;
#endif
- erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX);
+ erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
- erts_smp_atomic_init(&ps->no_avoided_wakeups, 0);
- erts_smp_atomic_init(&ps->no_avoided_interrupts, 0);
- erts_smp_atomic_init(&ps->no_interrupt_timed, 0);
+ erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0);
+ erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0);
+ erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0);
#endif
#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
handle_update_requests(ps);
@@ -2303,7 +2303,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
#if ERTS_POLL_USE_FALLBACK
ps->fallback_used = 0;
#endif
- erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
+ erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
erts_smp_spin_lock(&pollsets_lock);
ps->next = pollsets;
@@ -2449,7 +2449,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
pip->memory_size = size;
- pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds);
+ pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
#ifdef ERTS_SMP
pip->poll_set_size++; /* Wakeup pipe */
#endif
@@ -2507,9 +2507,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
pip->max_fds = max_fds;
#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
- pip->no_avoided_wakeups = erts_smp_atomic_read(&ps->no_avoided_wakeups);
- pip->no_avoided_interrupts = erts_smp_atomic_read(&ps->no_avoided_interrupts);
- pip->no_interrupt_timed = erts_smp_atomic_read(&ps->no_interrupt_timed);
+ pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups);
+ pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts);
+ pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed);
#endif
ERTS_POLLSET_UNLOCK(ps);
@@ -2529,7 +2529,7 @@ fatal_error(char *format, ...)
{
va_list ap;
- if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
+ if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
/*
* Crash dump writing and reception of sigusr1 (which will
* result in a crash dump) closes all file descriptors. This
@@ -2549,7 +2549,7 @@ fatal_error(char *format, ...)
static void
fatal_error_async_signal_safe(char *error_str)
{
- if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
+ if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
/* See comment above in fatal_error() */
return;
}
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index bafbbb0f6c..fd15635168 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -167,12 +167,12 @@ static int debug_log = 0;
#ifdef ERTS_SMP
erts_smp_atomic32_t erts_got_sigusr1;
#define ERTS_SET_GOT_SIGUSR1 \
- erts_smp_atomic32_set(&erts_got_sigusr1, 1)
+ erts_smp_atomic32_set_mb(&erts_got_sigusr1, 1)
#define ERTS_UNSET_GOT_SIGUSR1 \
- erts_smp_atomic32_set(&erts_got_sigusr1, 0)
+ erts_smp_atomic32_set_mb(&erts_got_sigusr1, 0)
static erts_smp_atomic32_t have_prepared_crash_dump;
#define ERTS_PREPARED_CRASH_DUMP \
- ((int) erts_smp_atomic32_xchg(&have_prepared_crash_dump, 1))
+ ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
#else
volatile int erts_got_sigusr1;
#define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1)
@@ -242,9 +242,9 @@ static int max_files = -1;
#ifdef ERTS_SMP
erts_smp_atomic32_t erts_break_requested;
#define ERTS_SET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
#define ERTS_UNSET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
#else
volatile int erts_break_requested = 0;
#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -364,7 +364,7 @@ Uint
erts_sys_misc_mem_sz(void)
{
Uint res = ERTS_CHK_IO_SZ();
- res += erts_smp_atomic_read(&sys_misc_mem_sz);
+ res += erts_smp_atomic_read_mb(&sys_misc_mem_sz);
return res;
}
@@ -509,9 +509,9 @@ erts_sys_pre_init(void)
#endif
}
#ifdef ERTS_SMP
- erts_smp_atomic32_init(&erts_break_requested, 0);
- erts_smp_atomic32_init(&erts_got_sigusr1, 0);
- erts_smp_atomic32_init(&have_prepared_crash_dump, 0);
+ erts_smp_atomic32_init_nob(&erts_break_requested, 0);
+ erts_smp_atomic32_init_nob(&erts_got_sigusr1, 0);
+ erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0);
#else
erts_break_requested = 0;
erts_got_sigusr1 = 0;
@@ -521,7 +521,7 @@ erts_sys_pre_init(void)
children_died = 0;
#endif
#endif /* USE_THREADS */
- erts_smp_atomic_init(&sys_misc_mem_sz, 0);
+ erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
}
void
@@ -553,7 +553,7 @@ erl_sys_init(void)
+ sizeof(CHILD_SETUP_PROG_NAME)
+ 1);
child_setup_prog = erts_alloc(ERTS_ALC_T_CS_PROG_PATH, csp_path_sz);
- erts_smp_atomic_add(&sys_misc_mem_sz, csp_path_sz);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, csp_path_sz);
sprintf(child_setup_prog,
"%s%c%s",
bindir,
@@ -1216,8 +1216,8 @@ static int spawn_init()
sys_sigset(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */
driver_data = (struct driver_data *)
erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data));
- erts_smp_atomic_add(&sys_misc_mem_sz,
- max_files * sizeof(struct driver_data));
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+ max_files * sizeof(struct driver_data));
for (i = 0; i < max_files; i++)
driver_data[i].pid = -1;
@@ -1925,8 +1925,8 @@ static void clear_fd_data(int fd)
{
if (fd_data[fd].sz > 0) {
erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fd_data[fd].buf);
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= fd_data[fd].sz);
- erts_smp_atomic_add(&sys_misc_mem_sz, -1*fd_data[fd].sz);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fd_data[fd].sz);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fd_data[fd].sz);
}
fd_data[fd].buf = NULL;
fd_data[fd].sz = 0;
@@ -2261,7 +2261,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
port_inp_failure(port_num, ready_fd, -1);
}
else {
- erts_smp_atomic_add(&sys_misc_mem_sz, h);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, h);
sys_memcpy(buf, cpos, bytes_left);
fd_data[ready_fd].buf = buf;
fd_data[ready_fd].sz = h;
@@ -2465,7 +2465,7 @@ erts_sys_putenv(char *buffer, int sep_ix)
#else
Uint sz = strlen(buffer)+1;
env = erts_alloc(ERTS_ALC_T_PUTENV_STR, sz);
- erts_smp_atomic_add(&sys_misc_mem_sz, sz);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, sz);
strcpy(env,buffer);
#endif
erts_smp_rwmtx_rwlock(&environ_rwmtx);
@@ -2504,8 +2504,8 @@ sys_init_io(void)
{
fd_data = (struct fd_data *)
erts_alloc(ERTS_ALC_T_FD_TAB, max_files * sizeof(struct fd_data));
- erts_smp_atomic_add(&sys_misc_mem_sz,
- max_files * sizeof(struct fd_data));
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+ max_files * sizeof(struct fd_data));
#ifdef USE_THREADS
#ifdef ERTS_SMP
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 074e2e247f..735c420d8e 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -309,9 +309,9 @@ struct ErtsPollSet_ {
#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#define ERTS_SET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
#define ERTS_UNSET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
#else
extern volatile int erts_break_requested;
#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -371,19 +371,19 @@ do { \
static ERTS_INLINE int
is_io_ready(ErtsPollSet ps)
{
- return erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
+ return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
}
static ERTS_INLINE void
woke_up(ErtsPollSet ps)
{
- if (erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
- erts_atomic32_cmpxchg(&ps->wakeup_state,
- ERTS_POLL_WOKEN_TIMEDOUT,
- ERTS_POLL_NOT_WOKEN);
+ if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
+ erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+ ERTS_POLL_WOKEN_TIMEDOUT,
+ ERTS_POLL_NOT_WOKEN);
#ifdef DEBUG
{
- erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
switch (wakeup_state) {
case ERTS_POLL_WOKEN_IO_READY:
case ERTS_POLL_WOKEN_INTR:
@@ -401,7 +401,7 @@ static ERTS_INLINE int
wakeup_cause(ErtsPollSet ps)
{
int res;
- erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
switch (wakeup_state) {
case ERTS_POLL_WOKEN_IO_READY:
res = 0;
@@ -439,7 +439,7 @@ poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
* by ResetEvent().
*/
ERTS_THR_MEMORY_BARRIER;
- if (erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
+ if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
return (DWORD) 0;
if (timeout > ERTS_AINT32_T_MAX) /* Also prevents DWORD overflow */
@@ -455,17 +455,17 @@ wake_poller(ErtsPollSet ps, int io_ready)
erts_aint32_t wakeup_state;
if (io_ready) {
/* We may set the event multiple times. This is, however, harmless. */
- wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
}
else {
ERTS_THR_MEMORY_BARRIER;
- wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
&& wakeup_state != ERTS_POLL_WOKEN_INTR) {
- erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state,
- ERTS_POLL_WOKEN_INTR,
- wakeup_state);
+ erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+ ERTS_POLL_WOKEN_INTR,
+ wakeup_state);
if (act == wakeup_state) {
wakeup_state = act;
break;
@@ -488,13 +488,13 @@ wake_poller(ErtsPollSet ps, int io_ready)
static ERTS_INLINE void
reset_io_ready(ErtsPollSet ps)
{
- erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+ erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
}
static ERTS_INLINE void
restore_io_ready(ErtsPollSet ps)
{
- erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
+ erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
}
/*
@@ -511,12 +511,12 @@ static ERTS_INLINE void
reset_interrupt(ErtsPollSet ps)
{
/* We need to keep io-ready if set */
- erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state);
+ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
&& wakeup_state != ERTS_POLL_NOT_WOKEN) {
- erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state,
- ERTS_POLL_NOT_WOKEN,
- wakeup_state);
+ erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+ ERTS_POLL_NOT_WOKEN,
+ wakeup_state);
if (wakeup_state == act)
break;
wakeup_state = act;
@@ -692,7 +692,7 @@ static void *break_waiter(void *param)
case WAIT_OBJECT_0:
ResetEvent(harr[0]);
erts_mtx_lock(&break_waiter_lock);
- erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+ erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
ERTS_THR_MEMORY_BARRIER;
SetEvent(break_happened_event);
erts_mtx_unlock(&break_waiter_lock);
@@ -700,7 +700,7 @@ static void *break_waiter(void *param)
case (WAIT_OBJECT_0+1):
ResetEvent(harr[1]);
erts_mtx_lock(&break_waiter_lock);
- erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+ erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_HALT);
ERTS_THR_MEMORY_BARRIER;
SetEvent(break_happened_event);
erts_mtx_unlock(&break_waiter_lock);
@@ -1153,7 +1153,7 @@ int erts_poll_wait(ErtsPollSet ps,
/*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
- if (timeout > 0 && !erts_atomic32_read(&break_waiter_state)) {
+ if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) {
HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
int num_h = 2;
@@ -1166,10 +1166,10 @@ int erts_poll_wait(ErtsPollSet ps,
}
ERTS_UNSET_BREAK_REQUESTED;
- if(erts_atomic32_read(&break_waiter_state)) {
+ if(erts_atomic32_read_nob(&break_waiter_state)) {
erts_mtx_lock(&break_waiter_lock);
- break_state = erts_atomic32_read(&break_waiter_state);
- erts_atomic32_set(&break_waiter_state,0);
+ break_state = erts_atomic32_read_nob(&break_waiter_state);
+ erts_atomic32_set_nob(&break_waiter_state,0);
ResetEvent(break_happened_event);
erts_mtx_unlock(&break_waiter_lock);
switch (break_state) {
@@ -1236,7 +1236,7 @@ int erts_poll_wait(ErtsPollSet ps,
erts_mtx_unlock(&w->mtx);
}
done:
- erts_smp_atomic32_set(&ps->timeout, ERTS_AINT32_T_MAX);
+ erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX);
*len = num;
ERTS_POLLSET_UNLOCK(ps);
HARDTRACEF(("Out erts_poll_wait"));
@@ -1316,11 +1316,11 @@ ErtsPollSet erts_poll_create_pollset(void)
ps->standby_wait_event = CreateManualEvent(FALSE);
ps->restore_events = 0;
- erts_atomic32_init(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+ erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
#ifdef ERTS_SMP
erts_smp_mtx_init(&ps->mtx, "pollset");
#endif
- erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX);
+ erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
HARDTRACEF(("Out erts_poll_create_pollset"));
return ps;
@@ -1372,7 +1372,7 @@ void erts_poll_init(void)
erts_mtx_init(&break_waiter_lock,"break_waiter_lock");
break_happened_event = CreateManualEvent(FALSE);
- erts_atomic32_init(&break_waiter_state, 0);
+ erts_atomic32_init_nob(&break_waiter_state, 0);
erts_thr_create(&thread, &break_waiter, NULL, NULL);
ERTS_UNSET_BREAK_REQUESTED;
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index a2159d063c..ce1d376a54 100644
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -198,7 +198,7 @@ Uint
erts_sys_misc_mem_sz(void)
{
Uint res = (Uint) erts_check_io_size();
- res += (Uint) erts_smp_atomic_read(&sys_misc_mem_sz);
+ res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz);
return res;
}
@@ -648,7 +648,7 @@ new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_
erts_smp_mtx_unlock(&sys_driver_data_lock);
return NULL;
}
- erts_smp_atomic_add(&sys_misc_mem_sz, dp->inBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize);
dp->outBufSize = 0;
dp->outbuf = NULL;
dp->port_num = port_num;
@@ -733,8 +733,8 @@ release_driver_data(DriverData* dp)
#endif
if (dp->inbuf != NULL) {
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize);
- erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->inBufSize);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize);
DRV_BUF_FREE(dp->inbuf);
dp->inBufSize = 0;
dp->inbuf = NULL;
@@ -742,8 +742,8 @@ release_driver_data(DriverData* dp)
ASSERT(dp->inBufSize == 0);
if (dp->outbuf != NULL) {
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
- erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
DRV_BUF_FREE(dp->outbuf);
dp->outBufSize = 0;
dp->outbuf = NULL;
@@ -1162,7 +1162,8 @@ spawn_init(void)
#endif
driver_data = (struct driver_data *)
erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data));
- erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data));
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+ max_files*sizeof(struct driver_data));
for (i = 0; i < max_files; i++)
driver_data[i].port_num = PORT_FREE;
@@ -1698,7 +1699,7 @@ create_child_process
static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL overlapped_io)
{
SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE};
- char pipe_name[128]; /* Name of pipe. */
+ char pipe_name[256]; /* Name of pipe. */
Uint calls;
/*
@@ -1735,9 +1736,9 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o
* Otherwise, create named pipes.
*/
- calls = (Uint) erts_smp_atomic_inctest(&pipe_creation_counter);
- sprintf(pipe_name, "\\\\.\\pipe\\erlang44_%d_%d",
- getpid(), calls);
+ calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter);
+ erts_snprintf(pipe_name, sizeof(pipe_name),
+ "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls);
DEBUGF(("Creating pipe %s\n", pipe_name));
sa.bInheritHandle = inheritRead;
@@ -2529,7 +2530,7 @@ output(ErlDrvData drv_data, char* buf, int len)
}
dp->outBufSize = pb+len;
- erts_smp_atomic_add(&sys_misc_mem_sz, dp->outBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize);
/*
* Store header bytes (if any).
@@ -2558,8 +2559,8 @@ output(ErlDrvData drv_data, char* buf, int len)
} else {
dp->out.ov.Offset += pb+len; /* For vanilla driver. */
/* XXX OffsetHigh should be changed too. */
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
- erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
DRV_BUF_FREE(dp->outbuf);
dp->outBufSize = 0;
dp->outbuf = NULL;
@@ -2673,9 +2674,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
error = ERROR_NOT_ENOUGH_MEMORY;
break; /* Break out of loop into error handler. */
}
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize);
- erts_smp_atomic_add(&sys_misc_mem_sz,
- dp->totalNeeded - dp->inBufSize);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+ dp->totalNeeded - dp->inBufSize);
dp->inBufSize = dp->totalNeeded;
dp->inbuf = new_buf;
}
@@ -2775,8 +2776,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event)
write... */
return;
}
- ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
- erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+ ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
DRV_BUF_FREE(dp->outbuf);
dp->outBufSize = 0;
dp->outbuf = NULL;
@@ -2926,8 +2927,8 @@ Preload* sys_preloaded(void)
(num_preloaded+1)*sizeof(Preload));
res_name = erts_alloc(ERTS_ALC_T_PRELOADED,
(num_preloaded+1)*sizeof(unsigned));
- erts_smp_atomic_add(&sys_misc_mem_sz,
- (num_preloaded+1)*sizeof(Preload)
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+ (num_preloaded+1)*sizeof(Preload)
+ (num_preloaded+1)*sizeof(unsigned));
for (i = 0; i < num_preloaded; i++) {
int n;
@@ -2939,7 +2940,7 @@ Preload* sys_preloaded(void)
n = GETWORD(data);
data += 2;
preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1);
- erts_smp_atomic_add(&sys_misc_mem_sz, n+1);
+ erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1);
sys_memcpy(preloaded[i].name, data, n);
preloaded[i].name[n] = '\0';
data += n;
@@ -3281,7 +3282,7 @@ erts_sys_pre_init(void)
#endif
}
#endif
- erts_smp_atomic_init(&sys_misc_mem_sz, 0);
+ erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
}
void noinherit_std_handle(DWORD type)
@@ -3310,7 +3311,7 @@ void erl_sys_init(void)
erts_smp_tsd_key_create(&win32_errstr_key);
InitializeCriticalSection(&htbc_lock);
#endif
- erts_smp_atomic_init(&pipe_creation_counter,0);
+ erts_smp_atomic_init_nob(&pipe_creation_counter,0);
/*
* Test if we have named pipes or not.
*/
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index 943c338794..1d73edd30b 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -33,9 +33,9 @@
#ifdef ERTS_SMP
erts_smp_atomic32_t erts_break_requested;
#define ERTS_SET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
#define ERTS_UNSET_BREAK_REQUESTED \
- erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0)
+ erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
#else
volatile int erts_break_requested = 0;
#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
index b869a4079c..8b34375980 100644
--- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h
+++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
@@ -82,15 +82,17 @@ typedef void* erts_cond;
#define NO_OF_BKTS ((Ulong) ALC_TEST0(0x102))
#define FIND_BKT(A, I) ((int) ALC_TEST2(0x103, (A), (I)))
-/* From erl_bestfit_alloc.c */
-#define IS_AOBF(A) ((Ulong) ALC_TEST1(0x200, (A)))
-#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(0x201, (A)))
-#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(0x202, (T)))
-#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(0x203, (T)))
-#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(0x204, (T)))
-#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(0x205, (T)))
-#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(0x206, (T)))
-#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(0x207, (T)))
+/* From erl_bestfit_alloc.c and erl_ao_firstfit_alloc.c */
+#define IS_AOBF(A) ((Ulong) ALC_TEST1(RBT_OP(0), (A)))
+#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(RBT_OP(1), (A)))
+#define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(RBT_OP(2), (T)))
+#define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(RBT_OP(3), (T)))
+#define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(RBT_OP(4), (T)))
+#define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(RBT_OP(5), (T)))
+#define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(RBT_OP(6), (T)))
+#define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(RBT_OP(7), (T)))
+#define IS_AOFF(A) ((Ulong) ALC_TEST1(RBT_OP(8), (A)))
+#define RBT_MAX_SZ(T) ((Ulong) ALC_TEST1(RBT_OP(9), (T)))
/* From erl_mseg.c */
#define HAVE_MSEG() ((int) ALC_TEST0(0x400))
diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c
index c84da97d35..6f35d3279b 100644
--- a/erts/emulator/test/alloc_SUITE_data/coalesce.c
+++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c
@@ -267,7 +267,7 @@ void
testcase_run(TestCaseState_t *tcs)
{
char *argv_org[] = {"-tmmbcs1024", "-tsbct2048", "-trmbcmt100", "-tas", NULL, NULL};
- char *alg[] = {"af", "gf", "bf", "aobf", NULL};
+ char *alg[] = {"af", "gf", "bf", "aobf", "aoff", NULL};
int i;
for (i = 0; alg[i]; i++) {
diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c
index c97e0aac1a..4e7f821baf 100644
--- a/erts/emulator/test/alloc_SUITE_data/rbtree.c
+++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c
@@ -34,6 +34,14 @@ typedef struct {
#define PRINT_TREE
#endif
+/* Ugly hack to steer the test code towards the right allocator */
+#define RBT_OP(CMD) (current_rbt_type_op_base + (CMD))
+static enum {
+ BESTFIT_OP_BASE = 0x200,
+ AO_FIRSTFIT_OP_BASE = 0x500
+}current_rbt_type_op_base;
+
+
#ifdef PRINT_TREE
#define INDENT_STEP 5
@@ -65,12 +73,11 @@ print_tree_aux(TestCaseState_t *tcs, RBT_t *x, int indent)
static void
-print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf)
+print_tree(TestCaseState_t *tcs, RBT_t *root)
{
- char *type = aobf ? "Size-Adress" : "Size";
- testcase_printf(tcs, " --- %s tree begin ---\r\n", type);
+ testcase_printf(tcs, " --- Tree begin ---\r\n");
print_tree_aux(tcs, root, 0);
- testcase_printf(tcs, " --- %s tree end ---\r\n", type);
+ testcase_printf(tcs, " --- Tree end ---\r\n");
}
#endif
@@ -78,7 +85,8 @@ print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf)
static RBT_t *
check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
{
- int i, max_i, address_order;
+ enum { BF, AOBF, AOFF }type;
+ int i, max_i;
char stk[128];
RBT_t *root, *x, *y, *res;
Ulong x_sz, y_sz, is_x_black;
@@ -86,11 +94,14 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
res = NULL;
- address_order = IS_AOBF(alc);
+ if (IS_AOBF(alc)) type = AOBF;
+ else if (IS_AOFF(alc)) type = AOFF;
+ else type = BF;
+
root = RBT_ROOT(alc);
#ifdef PRINT_TREE
- print_tree(tcs, root, address_order);
+ print_tree(tcs, root);
#endif
max_i = i = -1;
@@ -165,12 +176,18 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
if (y) {
y_sz = BLK_SZ(y);
ASSERT(tcs, RBT_PARENT(y) == x);
- if (address_order) {
+ switch (type) {
+ case AOBF:
ASSERT(tcs, y_sz < x_sz || (y_sz == x_sz && y < x));
- }
- else {
+ break;
+ case BF:
ASSERT(tcs, RBT_IS_TREE(y));
ASSERT(tcs, y_sz < x_sz);
+ break;
+ case AOFF:
+ ASSERT(tcs, y < x);
+ ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x));
+ break;
}
}
@@ -178,16 +195,22 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
if (y) {
y_sz = BLK_SZ(y);
ASSERT(tcs, RBT_PARENT(y) == x);
- if (address_order) {
+ switch (type) {
+ case AOBF:
ASSERT(tcs, y_sz > x_sz || (y_sz == x_sz && y > x));
- }
- else {
+ break;
+ case BF:
ASSERT(tcs, RBT_IS_TREE(y));
ASSERT(tcs, y_sz > x_sz);
+ break;
+ case AOFF:
+ ASSERT(tcs, y > x);
+ ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x));
+ break;
}
}
- if (!address_order) {
+ if (type == BF) {
Ulong l_sz;
RBTL_t *l = RBT_NEXT(x);
for (l = RBT_NEXT(x); l; l = RBT_NEXT(l)) {
@@ -202,13 +225,20 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
res = x;
else {
y_sz = BLK_SZ(res);
- if (address_order) {
+ switch (type) {
+ case AOBF:
if (x_sz < y_sz || (x_sz == y_sz && x < res))
res = x;
- }
- else {
- if (!res || x_sz < y_sz)
+ break;
+ case BF:
+ if (x_sz < y_sz)
res = x;
+ break;
+ case AOFF:
+ if (x < res) {
+ res = x;
+ }
+ break;
}
}
}
@@ -257,7 +287,7 @@ static void
test_it(TestCaseState_t *tcs)
{
int i;
- Allctr_t a = ((rbtree_test_data *) tcs->extra)->allocator;
+ Allctr_t* a = ((rbtree_test_data *) tcs->extra)->allocator;
void **blk = ((rbtree_test_data *) tcs->extra)->blk;
void **fence = ((rbtree_test_data *) tcs->extra)->fence;
Ulong min_blk_sz;
@@ -338,6 +368,7 @@ testcase_run(TestCaseState_t *tcs)
{
char *argv1[] = {"-tasbf", NULL};
char *argv2[] = {"-tasaobf", NULL};
+ char *argv3[] = {"-tasaoff", NULL};
Allctr_t *a;
rbtree_test_data *td;
@@ -355,6 +386,7 @@ testcase_run(TestCaseState_t *tcs)
testcase_printf(tcs, "Starting test of best fit...\n");
+ current_rbt_type_op_base = BESTFIT_OP_BASE;
td->allocator = a = START_ALC("rbtree_bf_", 0, argv1);
ASSERT(tcs, a);
@@ -371,6 +403,7 @@ testcase_run(TestCaseState_t *tcs)
testcase_printf(tcs, "Starting test of address order best fit...\n");
+ current_rbt_type_op_base = BESTFIT_OP_BASE;
td->allocator = a = START_ALC("rbtree_aobf_", 0, argv2);
ASSERT(tcs, a);
@@ -383,4 +416,19 @@ testcase_run(TestCaseState_t *tcs)
testcase_printf(tcs, "Address order best fit test succeeded!\n");
+ /* Address order first fit... */
+
+ testcase_printf(tcs, "Starting test of address order first fit...\n");
+
+ current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE;
+ td->allocator = a = START_ALC("rbtree_aoff_", 0, argv3);
+
+ ASSERT(tcs, a);
+
+ test_it(tcs);
+
+ STOP_ALC(a);
+ td->allocator = NULL;
+
+ testcase_printf(tcs, "Address order first fit test succeeded!\n");
}
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index 4e82381fba..459dc84565 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -33,7 +33,6 @@
%% erlang:external_size/1
%% size(Binary)
%% iolist_size/1
-%% concat_binary/1
%% split_binary/2
%% hash(Binary, N)
%% phash(Binary, N)
@@ -47,7 +46,7 @@
init_per_testcase/2, end_per_testcase/2,
copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1,
bad_list_to_binary/1, bad_binary_to_list/1,
- t_split_binary/1, bad_split/1, t_concat_binary/1,
+ t_split_binary/1, bad_split/1,
terms/1, terms_float/1, external_size/1, t_iolist_size/1,
t_hash/1,
bad_size/1,
@@ -68,7 +67,7 @@ suite() -> [{ct_hooks,[ts_install_cth]},
all() ->
[copy_terms, conversions, deep_lists, deep_bitstr_lists,
- t_split_binary, bad_split, t_concat_binary,
+ t_split_binary, bad_split,
bad_list_to_binary, bad_binary_to_list, terms,
terms_float, external_size, t_iolist_size,
bad_binary_to_term_2, safe_binary_to_term2,
@@ -93,7 +92,6 @@ init_per_group(_GroupName, Config) ->
end_per_group(_GroupName, Config) ->
Config.
-
init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
Config.
@@ -381,41 +379,6 @@ bad_split(Config) when is_list(Config) ->
bad_split(Bin, Pos) ->
{'EXIT',{badarg,_}} = (catch split_binary(Bin, Pos)).
-%% Tests concat_binary/2 and size/1.
-
-t_concat_binary(suite) -> [];
-t_concat_binary(Config) when is_list(Config) ->
- test_concat([]),
-
- test_concat([[]]),
- test_concat([[], []]),
- test_concat([[], [], []]),
-
- test_concat([[1], []]),
- test_concat([[], [2]]),
- test_concat([[], [3], []]),
-
- test_concat([[1, 2, 3], [4, 5, 6, 7]]),
- test_concat([[1, 2, 3], [4, 5, 6, 7], [9, 10]]),
-
- test_concat([lists:seq(0, 255), lists:duplicate(1024, $@),
- lists:duplicate(2048, $a),
- lists:duplicate(4000, $b)]),
- ok.
-
-test_concat(Lists) ->
- test_concat(Lists, 0, [], []).
-
-test_concat([List|Rest], Size, Combined, Binaries) ->
- ?line Bin = list_to_binary(List),
- ?line test_concat(Rest, Size+length(List), Combined++List, [Bin|Binaries]);
-test_concat([], Size, Combined, Binaries0) ->
- ?line Binaries = lists:reverse(Binaries0),
- ?line Bin = concat_binary(Binaries),
- ?line Size = size(Bin),
- ?line Size = iolist_size(Bin),
- ?line Combined = binary_to_list(Bin).
-
t_hash(doc) -> "Test hash/2 with different type of binaries.";
t_hash(Config) when is_list(Config) ->
test_hash([]),
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index 7795efe57e..559e540016 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -647,17 +647,11 @@ refc_dist_1() ->
%% Fun is passed in an exit signal. Wait until it is gone.
?line wait_until(fun () -> 4 =/= fun_refc(F2) end),
?line 3 = fun_refc(F2),
- erts_debug:set_internal_state(available_internal_state, true),
- ?line F_refc = case erts_debug:get_internal_state(force_heap_frags) of
- false -> 3;
- true -> 2 % GC after bif already decreased it
- end,
- ?line F_refc = fun_refc(F),
- erts_debug:set_internal_state(available_internal_state, false),
+ ?line true = erlang:garbage_collect(),
+ ?line 2 = fun_refc(F),
refc_dist_send(Node, F).
refc_dist_send(Node, F) ->
- ?line true = erlang:garbage_collect(),
?line Pid = spawn_link(Node,
fun() -> receive
{To,Fun} when is_function(Fun) ->
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 2b21fa58f4..461773114e 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -27,8 +27,9 @@
destructive_in_test_bif/1, guard_exceptions/1,
unary_plus/1, unary_minus/1, moving_labels/1]).
-export([fpe/1]).
+-export([otp_9422/1]).
--export([runner/2]).
+-export([runner/2, loop_runner/3]).
-export([f1/1, f2/2, f3/2, fn/1, fn/2, fn/3]).
-export([do_boxed_and_small/0]).
@@ -57,7 +58,8 @@ all() ->
trace_control_word, silent, silent_no_ms, ms_trace2,
ms_trace3, boxed_and_small, destructive_in_test_bif,
guard_exceptions, unary_plus, unary_minus, fpe,
- moving_labels];
+ moving_labels,
+ otp_9422];
true -> [not_run]
end.
@@ -208,6 +210,43 @@ test_3(Config) when is_list(Config) ->
?line collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]),
?line ok.
+otp_9422(doc) -> [];
+otp_9422(Config) when is_list(Config) ->
+ Laps = 1000,
+ ?line Fun1 = fun() -> otp_9422_tracee() end,
+ ?line P1 = spawn_link(?MODULE, loop_runner, [self(), Fun1, Laps]),
+ io:format("spawned ~p as tracee\n", [P1]),
+
+ ?line erlang:trace(P1, true, [call, silent]),
+
+ ?line Fun2 = fun() -> otp_9422_trace_changer() end,
+ ?line P2 = spawn_link(?MODULE, loop_runner, [self(), Fun2, Laps]),
+ io:format("spawned ~p as trace_changer\n", [P2]),
+
+ start_collect(P1),
+ start_collect(P2),
+
+ %%receive after 10*1000 -> ok end,
+
+ stop_collect(P1),
+ stop_collect(P2),
+ ok.
+
+otp_9422_tracee() ->
+ ?MODULE:f1(a),
+ ?MODULE:f1(b),
+ ?MODULE:f1(c).
+
+otp_9422_trace_changer() ->
+ Pat1 = [{[a], [], [{enable_trace, arity}]}],
+ ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat1),
+ Pat2 = [{[b], [], [{disable_trace, arity}]}],
+ ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat2).
+
+
+
+
+
bad_match_spec_bin(Config) when is_list(Config) ->
{'EXIT',{badarg,_}} = (catch ets:match_spec_run([1], <<>>)),
B0 = <<1,2>>,
@@ -932,6 +971,24 @@ runner(Collector, Fun) ->
Collector ! {gone, self()}
end.
+loop_runner(Collector, Fun, Laps) ->
+ receive
+ {go, Collector} ->
+ go
+ end,
+ loop_runner_cont(Collector, Fun, 0, Laps).
+
+loop_runner_cont(_Collector, _Fun, Laps, Laps) ->
+ receive
+ {done, Collector} ->
+ io:format("loop_runner ~p exit after ~p laps\n", [self(), Laps]),
+ Collector ! {gone, self()}
+ end;
+loop_runner_cont(Collector, Fun, N, Laps) ->
+ Fun(),
+ loop_runner_cont(Collector, Fun, N+1, Laps).
+
+
f1(X) ->
{X}.
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 91d695d979..9c31b7f78d 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -35,7 +35,7 @@
resource_takeover/1,
threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1,
is_checks/1,
- get_length/1, make_atom/1, make_string/1]).
+ get_length/1, make_atom/1, make_string/1, reverse_list_test/1]).
-export([many_args_100/100]).
@@ -60,7 +60,7 @@ all() ->
iolist_as_binary, resource, resource_binary,
resource_takeover, threading, send, send2, send3,
send_threaded, neg, is_checks, get_length, make_atom,
- make_string].
+ make_string,reverse_list_test].
groups() ->
[].
@@ -257,10 +257,48 @@ types(Config) when is_list(Config) ->
end,
[{},{ok},{{}},{[],{}},{1,2,3,4,5}]),
Stuff = [[],{},0,0.0,(1 bsl 100),(fun()-> ok end),make_ref(),self()],
- [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff],
+ [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff],
+
+ {IntSz, LongSz} = type_sizes(),
+ UintMax = (1 bsl (IntSz*8)) - 1,
+ IntMax = UintMax bsr 1,
+ IntMin = -(IntMax+1),
+ UlongMax = (1 bsl (LongSz*8)) - 1,
+ LongMax = UlongMax bsr 1,
+ LongMin = -(LongMax+1),
+ Uint64Max = (1 bsl 64) - 1,
+ Int64Max = Uint64Max bsr 1,
+ Int64Min = -(Int64Max+1),
+ Limits = [{IntMin,IntMax},{0,UintMax},{LongMin,LongMax},{0,UlongMax},{Int64Min,Int64Max},{0,Uint64Max}],
+ io:format("Limits = ~p\n", [Limits]),
+ lists:foreach(fun(I) ->
+ R1 = echo_int(I),
+ %%io:format("echo_int(~p) -> ~p\n", [I, R1]),
+ R2 = my_echo_int(I, Limits),
+ ?line R1 = R2,
+ ?line true = (R1 =:= R2),
+ ?line true = (R1 == R2)
+ end, int_list()),
+
?line verify_tmpmem(TmpMem),
ok.
+int_list() ->
+ Start = 1 bsl 200,
+ int_list([Start], -Start).
+int_list([N | _]=List, End) when N<End ->
+ List;
+int_list([N | _]=List, End) ->
+ int_list([N - (1 + (abs(N) div 3)) | List], End).
+
+my_echo_int(I, Limits) ->
+ lists:map(fun({Min,Max}) ->
+ if I < Min -> false;
+ I > Max -> false;
+ true -> I
+ end
+ end, Limits).
+
clone(X) ->
binary_to_term(term_to_binary(X)).
@@ -1164,15 +1202,23 @@ make_string(Config) when is_list(Config) ->
AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k],
?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}.
+reverse_list_test(Config) ->
+ ?line ensure_lib_loaded(Config, 1),
+ List = lists:seq(1,100),
+ RevList = lists:reverse(List),
+ ?line RevList = reverse_list(List),
+ ?line badarg = reverse_list(foo).
+
tmpmem() ->
case erlang:system_info({allocator,temp_alloc}) of
false -> undefined;
MemInfo ->
MSBCS = lists:foldl(
fun ({instance, _, L}, Acc) ->
+ {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L),
{value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
{value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
- [MBCS,SBCS | Acc]
+ [SBMBCS,MBCS,SBCS | Acc]
end,
[],
MemInfo),
@@ -1269,6 +1315,9 @@ send_blob_thread(_,_,_) -> ?nif_stub.
join_send_thread(_) -> ?nif_stub.
copy_blob(_) -> ?nif_stub.
send_term(_,_) -> ?nif_stub.
+reverse_list(_) -> ?nif_stub.
+echo_int(_) -> ?nif_stub.
+type_sizes() -> ?nif_stub.
nif_stub_error(Line) ->
exit({nif_not_loaded,module,?MODULE,line,Line}).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 0bb93daa33..bdf1549862 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -28,6 +28,7 @@
static int static_cntA; /* zero by default */
static int static_cntB = NIF_SUITE_LIB_VER * 100;
+static ERL_NIF_TERM atom_false;
static ERL_NIF_TERM atom_self;
static ERL_NIF_TERM atom_ok;
static ERL_NIF_TERM atom_join;
@@ -103,7 +104,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
msgenv_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.msgenv",
msgenv_dtor,
ERL_NIF_RT_CREATE, NULL);
-
+ atom_false = enif_make_atom(env,"false");
atom_self = enif_make_atom(env,"self");
atom_ok = enif_make_atom(env,"ok");
atom_join = enif_make_atom(env,"join");
@@ -481,6 +482,45 @@ error:
return enif_make_atom(env,"error");
}
+static ERL_NIF_TERM echo_int(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ int sint;
+ unsigned uint;
+ long slong;
+ unsigned long ulong;
+ ErlNifSInt64 sint64;
+ ErlNifUInt64 uint64;
+ ERL_NIF_TERM sint_term = atom_false, uint_term = atom_false;
+ ERL_NIF_TERM slong_term = atom_false, ulong_term = atom_false;
+ ERL_NIF_TERM sint64_term = atom_false, uint64_term = atom_false;
+
+ if (enif_get_int(env, argv[0], &sint)) {
+ sint_term = enif_make_int(env, sint);
+ }
+ if (enif_get_uint(env, argv[0], &uint)) {
+ uint_term = enif_make_uint(env, uint);
+ }
+ if (enif_get_long(env, argv[0], &slong)) {
+ slong_term = enif_make_long(env, slong);
+ }
+ if (enif_get_ulong(env, argv[0], &ulong)) {
+ ulong_term = enif_make_ulong(env, ulong);
+ }
+ if (enif_get_int64(env, argv[0], &sint64)) {
+ sint64_term = enif_make_int64(env, sint64);
+ }
+ if (enif_get_uint64(env, argv[0], &uint64)) {
+ uint64_term = enif_make_uint64(env, uint64);
+ }
+ return enif_make_list6(env, sint_term, uint_term, slong_term, ulong_term, sint64_term, uint64_term);
+}
+
+static ERL_NIF_TERM type_sizes(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+ return enif_make_tuple2(env, enif_make_int(env, sizeof(int)),
+ enif_make_int(env, sizeof(long)));
+}
+
static ERL_NIF_TERM tuple_2_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
int arity = -1;
@@ -1373,6 +1413,14 @@ static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
return enif_make_int(env, ret);
}
+static ERL_NIF_TERM reverse_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
+ ERL_NIF_TERM rev_list;
+
+ if(!enif_make_reverse_list(env, argv[0], &rev_list))
+ return enif_make_atom(env, "badarg");
+ return rev_list;
+}
+
static ErlNifFunc nif_funcs[] =
{
{"lib_version", 0, lib_version},
@@ -1417,7 +1465,10 @@ static ErlNifFunc nif_funcs[] =
{"send_blob_thread", 3, send_blob_thread},
{"join_send_thread", 1, join_send_thread},
{"copy_blob", 1, copy_blob},
- {"send_term", 2, send_term}
+ {"send_term", 2, send_term},
+ {"reverse_list",1, reverse_list},
+ {"echo_int", 1, echo_int},
+ {"type_sizes", 0, type_sizes}
};
ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload)
diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl
index 6615873392..ba0ba804ca 100644
--- a/erts/emulator/test/send_term_SUITE.erl
+++ b/erts/emulator/test/send_term_SUITE.erl
@@ -175,6 +175,10 @@ chk_temp_alloc() ->
%% Verify that we havn't got anything allocated by temp_alloc
lists:foreach(
fun ({instance, _, TI}) ->
+ ?line {value, {sbmbcs, SBMBCInfo}}
+ = lists:keysearch(sbmbcs, 1, TI),
+ ?line {value, {blocks, 0, _, _}}
+ = lists:keysearch(blocks, 1, SBMBCInfo),
?line {value, {mbcs, MBCInfo}}
= lists:keysearch(mbcs, 1, TI),
?line {value, {blocks, 0, _, _}}
diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h
index 2a0de4df9c..a2d7559f9d 100644
--- a/erts/epmd/src/epmd_int.h
+++ b/erts/epmd/src/epmd_int.h
@@ -240,6 +240,14 @@
#define put_int16(i, s) {((unsigned char*)(s))[0] = ((i) >> 8) & 0xff; \
((unsigned char*)(s))[1] = (i) & 0xff;}
+#if defined(__GNUC__)
+# define EPMD_INLINE __inline__
+#elif defined(__WIN32__)
+# define EPMD_INLINE __inline
+#else
+# define EPMD_INLINE
+#endif
+
/* ************************************************************************ */
/* Stuctures used by server */
@@ -295,6 +303,7 @@ typedef struct {
unsigned delay_write;
int max_conn;
int active_conn;
+ int select_fd_top;
char *progname;
Connection *conn;
Nodes nodes;
diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c
index 4d9b454f97..5debae26b6 100644
--- a/erts/epmd/src/epmd_srv.c
+++ b/erts/epmd/src/epmd_srv.c
@@ -80,6 +80,13 @@ static int reply(EpmdVars*,int,char *,int);
static void dbg_print_buf(EpmdVars*,char *,int);
static void print_names(EpmdVars*);
+static EPMD_INLINE void select_fd_set(EpmdVars* g, int fd)
+{
+ FD_SET(fd, &g->orig_read_mask);
+ if (fd >= g->select_fd_top) {
+ g->select_fd_top = fd + 1;
+ }
+}
void run(EpmdVars *g)
{
@@ -171,6 +178,7 @@ void run(EpmdVars *g)
g->max_conn -= num_sockets;
FD_ZERO(&g->orig_read_mask);
+ g->select_fd_top = 0;
for (i = 0; i < num_sockets; i++)
{
@@ -232,14 +240,14 @@ void run(EpmdVars *g)
dbg_perror(g,"failed to listen on socket");
epmd_cleanup_exit(g,1);
}
- FD_SET(listensock[i],&g->orig_read_mask);
+ select_fd_set(g, listensock[i]);
}
dbg_tty_printf(g,2,"entering the main select() loop");
select_again:
while(1)
- {
+ {
fd_set read_mask = g->orig_read_mask;
struct timeval timeout;
int ret;
@@ -251,7 +259,8 @@ void run(EpmdVars *g)
timeout.tv_sec = (g->packet_timeout < IDLE_TIMEOUT) ? 1 : IDLE_TIMEOUT;
timeout.tv_usec = 0;
- if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0) {
+ if ((ret = select(g->select_fd_top,
+ &read_mask, (fd_set *)0,(fd_set *)0,&timeout)) < 0) {
dbg_perror(g,"error in select ");
switch (errno) {
case EAGAIN:
@@ -821,7 +830,7 @@ static int conn_open(EpmdVars *g,int fd)
s = &g->conn[i];
/* From now on we want to know if there are data to be read */
- FD_SET(fd, &g->orig_read_mask);
+ select_fd_set(g, fd);
s->fd = fd;
s->open = EPMD_TRUE;
@@ -886,6 +895,7 @@ int epmd_conn_close(EpmdVars *g,Connection *s)
dbg_tty_printf(g,2,"closing connection on file descriptor %d",s->fd);
FD_CLR(s->fd,&g->orig_read_mask);
+ /* we don't bother lowering g->select_fd_top */
close(s->fd); /* Sometimes already closed but close anyway */
s->open = EPMD_FALSE;
if (s->buf != NULL) { /* Should never be NULL but test anyway */
@@ -1115,7 +1125,7 @@ static Node *node_reg2(EpmdVars *g,
node->extralen = extralen;
memcpy(node->extra,extra,extralen);
strcpy(node->symname,name);
- FD_SET(fd,&g->orig_read_mask);
+ select_fd_set(g, fd);
if (highvsn == 0) {
dbg_tty_printf(g,1,"registering '%s:%d', port %d",
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index 7e04724f5a..2bd576d8e8 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -64,6 +64,7 @@
static const char plusM_au_allocs[]= {
'u', /* all alloc_util allocators */
'B', /* binary_alloc */
+ 'C', /* sbmbc_alloc */
'D', /* std_alloc */
'E', /* ets_alloc */
'H', /* eheap_alloc */
@@ -93,6 +94,8 @@ static char *plusM_au_alloc_switches[] = {
"rsbcst",
"sbct",
"smbcs",
+ "sbmbcs",
+ "sbmbct",
NULL
};
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
index 1caf4d0567..0f3c26f1df 100644
--- a/erts/include/internal/ethr_atomics.h
+++ b/erts/include/internal/ethr_atomics.h
@@ -1,7 +1,16 @@
/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -18,709 +27,8818 @@
*/
/*
- * Description: The ethread atomic API
+ * Description: The ethread atomics API
* Author: Rickard Green
*/
-#ifndef ETHR_ATOMIC_H__
-#define ETHR_ATOMIC_H__
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+ * - cmpxchg
+ * - xchg
+ * - set
+ * - init
+ * - add_read
+ * - read
+ * - inc_read
+ * - dec_read
+ * - add
+ * - inc
+ * - dec
+ * - read_band
+ * - read_bor
+ *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+ * - cmpxchg
+ * - set
+ * - read
+ * - init
+ *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * memory barrier semantics:
+ * - rb (read barrier)
+ * - wb (write barrier)
+ * - acqb (acquire barrier)
+ * - relb (release barrier)
+ * - mb (full memory barrier)
+ *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ * ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ * ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ * ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ * and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ * either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ * (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ * A macro informing about the presence of the native
+ * implementation:
+ *
+ * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ * A macro naming (a string constant) the implementation:
+ *
+ * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ * Each implemented native atomic function has to
+ * be accompanied by a defined macro on the following
+ * form informing about its presence:
+ *
+ * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ * A (sparc-v9 style) membar macro:
+ *
+ * - ETHR_MEMBAR(B)
+ *
+ * Which takes a combination of the following macros
+ * or:ed (using |) together:
+ *
+ * - ETHR_LoadLoad
+ * - ETHR_LoadStore
+ * - ETHR_StoreLoad
+ * - ETHR_StoreStore
+ *
+ */
-#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-# define ETHR_NEED_ATOMIC_PROTOTYPES__
-#endif
+#ifndef ETHR_ATOMICS_H__
+#define ETHR_ATOMICS_H__
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
+#undef ETHR_AMC_FALLBACK__
+#undef ETHR_AMC_NO_ATMCS__
+#undef ETHR_AMC_ATMC_T__
+#undef ETHR_AMC_ATMC_FUNC__
+
+/* -- 32-bit atomics -- */
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+#undef ETHR_NATMC32_ADDR_FUNC__
+#undef ETHR_NATMC32_BITS__
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+# define ETHR_NEED_NATMC32_ADDR
+# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic32_t;
+# define ETHR_NAINT32_T__ ethr_sint32_t
+# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_NATMC32_BITS__ 32
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# define ETHR_NEED_NATMC64_ADDR
+#ifdef ETHR_BIGENDIAN
+# define ETHR_NATMC32_ADDR_FUNC__(VAR) \
+ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+# define ETHR_NATMC32_ADDR_FUNC__(VAR) \
+ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic32_t;
+# define ETHR_NAINT32_T__ ethr_sint64_t
+# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC32_BITS__ 64
+#else
/*
- * No native atomic implementation available. :(
+ * No native atomics usable for 32-bits atomics :(
* Use fallback...
*/
typedef ethr_sint32_t ethr_atomic32_t;
-typedef ethr_sint_t ethr_atomic_t;
-#else
-/*
- * Map ethread native atomics to ethread API atomics.
- *
- * We do at least have a native atomic implementation that
- * can handle integers of a size larger than or equal to
- * the size of pointers.
- */
+#endif
-/* -- Pointer size atomics -- */
+#undef ETHR_ATMC32_INLINE__
+#ifdef ETHR_NATMC32_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC32_INLINE__
+# endif
+# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_ATMC32_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
+# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC32_FUNC__
+#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+
+/* -- Word size atomics -- */
+
+#undef ETHR_NEED_NATMC32_ADDR
+#undef ETHR_NEED_NATMC64_ADDR
#undef ETHR_NAINT_T__
#undef ETHR_NATMC_FUNC__
#undef ETHR_NATMC_ADDR_FUNC__
-#if ETHR_SIZEOF_PTR == 8
-# if defined(ETHR_HAVE_NATIVE_ATOMIC64)
-# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
+#undef ETHR_NATMC_BITS__
+#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
typedef ethr_native_atomic64_t ethr_atomic_t;
-# define ETHR_NAINT_T__ ethr_sint64_t
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-# else
-# error "Missing native atomic implementation"
+# define ETHR_NAINT_T__ ethr_sint64_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC_BITS__ 64
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC32_ADDR
# endif
-#elif ETHR_SIZEOF_PTR == 4
# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
-# ifdef ETHR_HAVE_NATIVE_ATOMIC32
typedef ethr_native_atomic32_t ethr_atomic_t;
-# define ETHR_NAINT_T__ ethr_sint32_t
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-# elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# define ETHR_NAINT_T__ ethr_sint32_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_NATMC_BITS__ 32
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+#ifdef ETHR_BIGENDIAN
+# define ETHR_NATMC_ADDR_FUNC__(VAR) \
+ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+# define ETHR_NATMC_ADDR_FUNC__(VAR) \
+ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
typedef ethr_native_atomic64_t ethr_atomic_t;
-# define ETHR_NATMC_T__ ethr_native_atomic64_t
-# define ETHR_NAINT_T__ ethr_sint64_t
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC_T__ ethr_native_atomic64_t
+# define ETHR_NAINT_T__ ethr_sint64_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC_BITS__ 64
+#else
+/*
+ * No native atomics usable for pointer size atomics :(
+ * Use fallback...
+ */
+
+# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 2
+# define ETHR_AMC_SINT_T__ ethr_sint32_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+typedef struct {
+ ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+typedef struct {
+ ethr_amc_t amc;
+ ethr_sint_t sint;
+} ethr_atomic_t;
+# else /* locked fallback */
+typedef ethr_sint_t ethr_atomic_t;
+# endif
+#endif
+
+#undef ETHR_ATMC_INLINE__
+#ifdef ETHR_NATMC_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC_INLINE__
+# endif
+# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC_FUNC_NAME_
+# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC_FUNC__
+#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+
+/* -- Double word atomics -- */
+
+#undef ETHR_SU_DW_NAINT_T__
+#undef ETHR_SU_DW_NATMC_FUNC__
+#undef ETHR_SU_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_FUNC__
+#undef ETHR_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_BITS__
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+# define ETHR_NEED_DW_NATMC_ADDR
+# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
+# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
+# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
+# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
+# if ETHR_SIZEOF_PTR == 8
+# define ETHR_DW_NATMC_BITS__ 128
+# elif ETHR_SIZEOF_PTR == 4
+# define ETHR_DW_NATMC_BITS__ 64
# else
-# error "Missing native atomic implementation"
+# error "Word size not supported"
+# endif
+# ifdef ETHR_NATIVE_SU_DW_SINT_T
+# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
# endif
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \
+ ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
+# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
+# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
+# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_DW_NATMC_BITS__ 64
#endif
-/* -- 32-bit atomics -- */
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__
+#else
+#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X
+#endif
-#undef ETHR_NAINT32_T__
-#undef ETHR_NATMC32_FUNC__
-#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
-typedef ethr_native_atomic32_t ethr_atomic32_t;
-# define ETHR_NAINT32_T__ ethr_sint32_t
-# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
-#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
-typedef ethr_native_atomic64_t ethr_atomic32_t;
-# define ETHR_NAINT32_T__ ethr_sint64_t
-# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+#if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+# define ETHR_NEED_DW_FALLBACK__
+#endif
+
+#if defined(ETHR_NEED_DW_FALLBACK__)
+/*
+ * No native atomics usable for double word atomics :(
+ * Use fallback...
+ */
+
+# ifndef ETHR_AMC_FALLBACK__
+# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 1
+# define ETHR_AMC_SINT_T__ ethr_sint_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 2
+# define ETHR_AMC_SINT_T__ ethr_sint32_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+# endif
+# ifdef ETHR_AMC_FALLBACK__
+typedef struct {
+ ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+# endif
+# endif
+
+typedef struct {
+#ifdef ETHR_AMC_FALLBACK__
+ ethr_amc_t amc;
+#endif
+ ethr_sint_t sint[2];
+} ethr_dw_atomic_fallback_t;
+
+#endif
+
+typedef union {
+#ifdef ETHR_NATIVE_DW_ATOMIC_T__
+ ETHR_NATIVE_DW_ATOMIC_T__ native;
+#endif
+#ifdef ETHR_NEED_DW_FALLBACK__
+ ethr_dw_atomic_fallback_t fallback;
+#endif
+ ethr_sint_t sint[2];
+} ethr_dw_atomic_t;
+
+typedef union {
+#ifdef ETHR_SU_DW_NAINT_T__
+ ETHR_SU_DW_NAINT_T__ dw_sint;
+#endif
+ ethr_sint_t sint[2];
+} ethr_dw_sint_t;
+
+#ifdef ETHR_BIGENDIAN
+# define ETHR_DW_SINT_LOW_WORD 1
+# define ETHR_DW_SINT_HIGH_WORD 0
#else
-# error "Missing native atomic implementation"
+# define ETHR_DW_SINT_LOW_WORD 0
+# define ETHR_DW_SINT_HIGH_WORD 1
#endif
+#undef ETHR_DW_ATMC_INLINE__
+#ifdef ETHR_DW_NATMC_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC32_INLINE__
+# endif
+# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
#endif
-#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
-ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *);
-void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
-void ethr_atomic_inc(ethr_atomic_t *);
-void ethr_atomic_dec(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
-void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_dec_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-
-ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *);
-void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t);
-void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *);
-void ethr_atomic32_inc(ethr_atomic32_t *);
-void ethr_atomic32_dec(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t);
-void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *,
- ethr_sint32_t,
- ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *);
-void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t);
-void ethr_atomic32_dec_relb(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *);
-ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *,
- ethr_sint32_t,
- ethr_sint32_t);
-ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *,
- ethr_sint32_t,
- ethr_sint32_t);
+#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_DW_ATMC_PROTOTYPES__
#endif
-int ethr_init_atomics(void);
+#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
+# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_DW_ATMC_FUNC__
+#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)
+
+#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
+int ethr_have_native_dw_atomic(void);
+#endif
+#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+static ETHR_INLINE int
+ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
+{
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__;
+#elif defined(ETHR_DW_NATMC_BITS__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+#endif
+
+/* -- Misc -- */
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+/*
+ * Unusual values are used by read() fallbacks implemented via cmpxchg().
+ * We want to use an unusual value in hope that it is more efficient
+ * not to match the value in memory.
+ *
+ * - Negative integer values are probably more unusual.
+ * - Very large absolute integer values are probably more unusual.
+ * - Odd pointers are probably more unusual (only char pointers can be odd).
+ */
+# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
+# if ETHR_SIZEOF_PTR == 4
+# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
+# elif ETHR_SIZEOF_PTR == 8
+# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
+# else
+# error "Word size not supported"
+# endif
+# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
+# error "No ethr_native_dw_atomic_addr() available"
+# endif
+# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
+# error "No ethr_native_atomic32_addr() available"
+# endif
+# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
+# error "No ethr_native_atomic64_addr() available"
+# endif
+#endif
+
+#if defined(__GNUC__)
+# ifndef ETHR_COMPILER_BARRIER
+# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+# endif
+#elif defined(ETHR_WIN32_THREADS)
+# ifndef ETHR_COMPILER_BARRIER
+# include <intrin.h>
+# pragma intrinsic(_ReadWriteBarrier)
+# define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+# endif
+#endif
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
+void ethr_compiler_barrier_fallback(void);
+#ifndef ETHR_COMPILER_BARRIER
+# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
+#endif
+
+int ethr_init_atomics(void);
+
+/* info */
+char **ethr_native_atomic32_ops(void);
+char **ethr_native_atomic64_ops(void);
+char **ethr_native_dw_atomic_ops(void);
+char **ethr_native_su_dw_atomic_ops(void);
+
+#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
/*
- * Fallbacks for atomics used in absence of a native implementation.
+ * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
+ * i.e. when no native atomic implementation exist and only our lock
+ * based atomic fallback is used, a noop is sufficient.
*/
+# undef ETHR_MEMORY_BARRIER
+# undef ETHR_WRITE_MEMORY_BARRIER
+# undef ETHR_READ_MEMORY_BARRIER
+# undef ETHR_READ_DEPEND_MEMORY_BARRIER
+# undef ETHR_MEMBAR
+# define ETHR_MEMBAR(B) do { } while (0)
+#endif
+
+#ifndef ETHR_MEMBAR
+# error "No ETHR_MEMBAR defined"
+#endif
-#define ETHR_ATOMIC_ADDR_BITS 10
-#define ETHR_ATOMIC_ADDR_SHIFT 6
+#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
+#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
+#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
+#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
+# undef ETHR_ORDERED_READ_DEPEND
+#else
+# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+# define ETHR_ORDERED_READ_DEPEND
+#endif
-typedef struct {
- union {
- ethr_spinlock_t lck;
- char buf[ETHR_CACHE_LINE_SIZE];
- } u;
-} ethr_atomic_protection_t;
-extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+/* ---------- Double word size atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__
+ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var);
+int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+#endif
+#endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+ && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_DW_NATMC_BITS__)
+# error "Missing native atomic implementation"
+#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
+# define ETHR_HAVE_DW_NATMC_CMPXCHG 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
+# define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
+# define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
+# define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
+# define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+# define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET
+# undef ETHR_HAVE_DW_NATMC_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
+# define ETHR_HAVE_DW_NATMC_SET 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+# define ETHR_HAVE_SU_DW_NATMC_SET 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET_RB
+# undef ETHR_HAVE_DW_NATMC_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
+# define ETHR_HAVE_DW_NATMC_SET_RB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
+# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET_WB
+# undef ETHR_HAVE_DW_NATMC_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
+# define ETHR_HAVE_DW_NATMC_SET_WB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
+# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
+# undef ETHR_HAVE_DW_NATMC_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
+# define ETHR_HAVE_DW_NATMC_SET_ACQB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
+# undef ETHR_HAVE_DW_NATMC_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
+# define ETHR_HAVE_DW_NATMC_SET_RELB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
+# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_SET_MB
+# undef ETHR_HAVE_DW_NATMC_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
+# define ETHR_HAVE_DW_NATMC_SET_MB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
+# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ
+# undef ETHR_HAVE_DW_NATMC_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
+# define ETHR_HAVE_DW_NATMC_READ 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+# define ETHR_HAVE_SU_DW_NATMC_READ 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ_RB
+# undef ETHR_HAVE_DW_NATMC_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
+# define ETHR_HAVE_DW_NATMC_READ_RB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
+# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ_WB
+# undef ETHR_HAVE_DW_NATMC_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
+# define ETHR_HAVE_DW_NATMC_READ_WB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
+# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
+# undef ETHR_HAVE_DW_NATMC_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
+# define ETHR_HAVE_DW_NATMC_READ_ACQB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
+# undef ETHR_HAVE_DW_NATMC_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
+# define ETHR_HAVE_DW_NATMC_READ_RELB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
+# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_READ_MB
+# undef ETHR_HAVE_DW_NATMC_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
+# define ETHR_HAVE_DW_NATMC_READ_MB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
+# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT
+# undef ETHR_HAVE_DW_NATMC_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
+# define ETHR_HAVE_DW_NATMC_INIT 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
+# define ETHR_HAVE_SU_DW_NATMC_INIT 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
+# undef ETHR_HAVE_DW_NATMC_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
+# define ETHR_HAVE_DW_NATMC_INIT_RB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
+# undef ETHR_HAVE_DW_NATMC_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
+# define ETHR_HAVE_DW_NATMC_INIT_WB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
+# undef ETHR_HAVE_DW_NATMC_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
+# define ETHR_HAVE_DW_NATMC_INIT_ACQB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
+# undef ETHR_HAVE_DW_NATMC_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
+# define ETHR_HAVE_DW_NATMC_INIT_RELB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
+# undef ETHR_HAVE_DW_NATMC_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
+# define ETHR_HAVE_DW_NATMC_INIT_MB 1
+# endif
+# ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
+# endif
+#elif ETHR_DW_NATMC_BITS__ == 64
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
+# undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+# define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET
+# undef ETHR_HAVE_SU_DW_NATMC_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+# define ETHR_HAVE_SU_DW_NATMC_SET 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET_RB
+# undef ETHR_HAVE_SU_DW_NATMC_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+# define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET_WB
+# undef ETHR_HAVE_SU_DW_NATMC_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+# define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET_ACQB
+# undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET_RELB
+# undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+# define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_SET_MB
+# undef ETHR_HAVE_SU_DW_NATMC_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+# define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ
+# undef ETHR_HAVE_SU_DW_NATMC_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+# define ETHR_HAVE_SU_DW_NATMC_READ 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ_RB
+# undef ETHR_HAVE_SU_DW_NATMC_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+# define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ_WB
+# undef ETHR_HAVE_SU_DW_NATMC_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+# define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ_ACQB
+# undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ_RELB
+# undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+# define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_READ_MB
+# undef ETHR_HAVE_SU_DW_NATMC_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+# define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT
+# undef ETHR_HAVE_SU_DW_NATMC_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+# define ETHR_HAVE_SU_DW_NATMC_INIT 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT_RB
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT_WB
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT_ACQB
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT_RELB
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_DW_NATMC_INIT_MB
+# undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+# define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
+# endif
+#else
+# error "Invalid native atomic size"
+#endif
+
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
- & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)
+#if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \
+ && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \
+ && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \
+ && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \
+ && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \
+ && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB))
+# error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_DW_NATMC_READ)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+ VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+do { \
+ VAL.sint[0] = (ethr_sint_t) 0; \
+ VAL.sint[1] = (ethr_sint_t) 0; \
+} while (0)
+#endif
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \
-do { \
- ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \
- ethr_spin_lock(slp__); \
- { EXPS; } \
- ethr_spin_unlock(slp__); \
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+ int res__; \
+ ethr_dw_sint_t AVAL, exp_act__; \
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \
+ do { \
+ AVAL.sint[0] = exp_act__.sint[0]; \
+ AVAL.sint[1] = exp_act__.sint[1]; \
+ { OPS; } \
+ res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \
+ } while (__builtin_expect(res__ == 0, 0)); \
} while (0)
+
+#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+
+#if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \
+ && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \
+ && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \
+ && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \
+ && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \
+ && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB))
+# error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ((ETHR_SU_DW_NAINT_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+ ETHR_SU_DW_NAINT_T__ AVAL; \
+ ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \
+ act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
+ do { \
+ exp__ = act__; \
+ AVAL = (ETHR_SU_DW_NAINT_T__) act__; \
+ { OPS; } \
+ new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \
+ act__ = CMPXCHG(VAR, new__, exp__); \
+ } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
+
+
+#else
+# error "?!?"
+#endif
+
+
+
+/* --- addr() --- */
+
+static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var)
+{
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+ return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native));
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); }
+#endif
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_StoreStore);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_StoreStore);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_StoreStore);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_StoreStore);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NAINT_T__ act;
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NAINT_T__ act;
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ res = (act == old_val->dw_sint);
+ old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); }
+#endif
+
+ return res;
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_rb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_wb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_acqb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_relb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+ ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+ ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+ ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_mb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); }
+#endif
+
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+ ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+ ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+ ethr_dw_sint_t tmp;
+ tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+ val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+ (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); }
+#endif
+
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#else
+ ETHR_DW_ATMC_FUNC__(set)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#else
+ ETHR_DW_ATMC_FUNC__(set_wb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_DW_ATMC_FUNC__(set_acqb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#else
+ ETHR_DW_ATMC_FUNC__(set_relb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+ ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+ ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+ ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+ ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_DW_ATMC_FUNC__(set_mb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); }
+#endif
+
+}
+
+#endif /* ETHR_DW_ATMC_INLINE__ */
+
+
+/* ---------- Word size atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_ATMC_PROTOTYPES__
+ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var);
+void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_inc(ethr_atomic_t *var);
+void ethr_atomic_inc_rb(ethr_atomic_t *var);
+void ethr_atomic_inc_wb(ethr_atomic_t *var);
+void ethr_atomic_inc_acqb(ethr_atomic_t *var);
+void ethr_atomic_inc_relb(ethr_atomic_t *var);
+void ethr_atomic_inc_mb(ethr_atomic_t *var);
+void ethr_atomic_dec(ethr_atomic_t *var);
+void ethr_atomic_dec_rb(ethr_atomic_t *var);
+void ethr_atomic_dec_wb(ethr_atomic_t *var);
+void ethr_atomic_dec_acqb(ethr_atomic_t *var);
+void ethr_atomic_dec_relb(ethr_atomic_t *var);
+void ethr_atomic_dec_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val);
+#endif /* ETHR_NEED_ATMC_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+ && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_NATMC_BITS__)
+# error "Missing native atomic implementation"
+#elif ETHR_NATMC_BITS__ == 64
+# undef ETHR_HAVE_NATMC_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+# define ETHR_HAVE_NATMC_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+# define ETHR_HAVE_NATMC_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+# define ETHR_HAVE_NATMC_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+# define ETHR_HAVE_NATMC_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+# define ETHR_HAVE_NATMC_XCHG 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+# define ETHR_HAVE_NATMC_XCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+# define ETHR_HAVE_NATMC_XCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+# define ETHR_HAVE_NATMC_XCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+# define ETHR_HAVE_NATMC_XCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+# define ETHR_HAVE_NATMC_XCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+# define ETHR_HAVE_NATMC_SET 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+# define ETHR_HAVE_NATMC_SET_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+# define ETHR_HAVE_NATMC_SET_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+# define ETHR_HAVE_NATMC_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+# define ETHR_HAVE_NATMC_SET_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+# define ETHR_HAVE_NATMC_SET_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+# define ETHR_HAVE_NATMC_INIT 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+# define ETHR_HAVE_NATMC_INIT_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+# define ETHR_HAVE_NATMC_INIT_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+# define ETHR_HAVE_NATMC_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+# define ETHR_HAVE_NATMC_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+# define ETHR_HAVE_NATMC_INIT_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+# define ETHR_HAVE_NATMC_ADD_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+# define ETHR_HAVE_NATMC_READ 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+# define ETHR_HAVE_NATMC_READ_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+# define ETHR_HAVE_NATMC_READ_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+# define ETHR_HAVE_NATMC_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+# define ETHR_HAVE_NATMC_READ_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+# define ETHR_HAVE_NATMC_READ_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+# define ETHR_HAVE_NATMC_INC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+# define ETHR_HAVE_NATMC_INC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+# define ETHR_HAVE_NATMC_INC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+# define ETHR_HAVE_NATMC_INC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+# define ETHR_HAVE_NATMC_DEC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+# define ETHR_HAVE_NATMC_ADD 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+# define ETHR_HAVE_NATMC_ADD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+# define ETHR_HAVE_NATMC_ADD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+# define ETHR_HAVE_NATMC_ADD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+# define ETHR_HAVE_NATMC_ADD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+# define ETHR_HAVE_NATMC_ADD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+# define ETHR_HAVE_NATMC_INC 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+# define ETHR_HAVE_NATMC_INC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+# define ETHR_HAVE_NATMC_INC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+# define ETHR_HAVE_NATMC_INC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+# define ETHR_HAVE_NATMC_INC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+# define ETHR_HAVE_NATMC_INC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+# define ETHR_HAVE_NATMC_DEC 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+# define ETHR_HAVE_NATMC_DEC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+# define ETHR_HAVE_NATMC_DEC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+# define ETHR_HAVE_NATMC_DEC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+# define ETHR_HAVE_NATMC_DEC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+# define ETHR_HAVE_NATMC_DEC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+# define ETHR_HAVE_NATMC_AND_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+# define ETHR_HAVE_NATMC_OR_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
+# endif
+#elif ETHR_NATMC_BITS__ == 32
+# undef ETHR_HAVE_NATMC_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+# define ETHR_HAVE_NATMC_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+# define ETHR_HAVE_NATMC_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+# define ETHR_HAVE_NATMC_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+# define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+# define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+# define ETHR_HAVE_NATMC_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+# define ETHR_HAVE_NATMC_XCHG 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+# define ETHR_HAVE_NATMC_XCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+# define ETHR_HAVE_NATMC_XCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+# define ETHR_HAVE_NATMC_XCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+# define ETHR_HAVE_NATMC_XCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_XCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+# define ETHR_HAVE_NATMC_XCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+# define ETHR_HAVE_NATMC_SET 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+# define ETHR_HAVE_NATMC_SET_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+# define ETHR_HAVE_NATMC_SET_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+# define ETHR_HAVE_NATMC_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+# define ETHR_HAVE_NATMC_SET_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+# define ETHR_HAVE_NATMC_SET_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+# define ETHR_HAVE_NATMC_INIT 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+# define ETHR_HAVE_NATMC_INIT_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+# define ETHR_HAVE_NATMC_INIT_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+# define ETHR_HAVE_NATMC_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+# define ETHR_HAVE_NATMC_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+# define ETHR_HAVE_NATMC_INIT_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+# define ETHR_HAVE_NATMC_ADD_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+# define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+# define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+# define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+# define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+# define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+# define ETHR_HAVE_NATMC_READ 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+# define ETHR_HAVE_NATMC_READ_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+# define ETHR_HAVE_NATMC_READ_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+# define ETHR_HAVE_NATMC_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+# define ETHR_HAVE_NATMC_READ_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+# define ETHR_HAVE_NATMC_READ_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+# define ETHR_HAVE_NATMC_INC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+# define ETHR_HAVE_NATMC_INC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+# define ETHR_HAVE_NATMC_INC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+# define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+# define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+# define ETHR_HAVE_NATMC_INC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+# define ETHR_HAVE_NATMC_DEC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+# define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+# define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+# define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+# define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+# define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+# define ETHR_HAVE_NATMC_ADD 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+# define ETHR_HAVE_NATMC_ADD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+# define ETHR_HAVE_NATMC_ADD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+# define ETHR_HAVE_NATMC_ADD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+# define ETHR_HAVE_NATMC_ADD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_ADD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+# define ETHR_HAVE_NATMC_ADD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+# define ETHR_HAVE_NATMC_INC 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+# define ETHR_HAVE_NATMC_INC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+# define ETHR_HAVE_NATMC_INC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+# define ETHR_HAVE_NATMC_INC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+# define ETHR_HAVE_NATMC_INC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_INC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+# define ETHR_HAVE_NATMC_INC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+# define ETHR_HAVE_NATMC_DEC 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+# define ETHR_HAVE_NATMC_DEC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+# define ETHR_HAVE_NATMC_DEC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+# define ETHR_HAVE_NATMC_DEC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+# define ETHR_HAVE_NATMC_DEC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_DEC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+# define ETHR_HAVE_NATMC_DEC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+# define ETHR_HAVE_NATMC_AND_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+# define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+# define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+# define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+# define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_AND_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+# define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+# define ETHR_HAVE_NATMC_OR_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+# define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+# define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+# define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+# define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC_OR_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+# define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
+# endif
+#else
+# error "Invalid native atomic size"
#endif
+#if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \
+ && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \
+ && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \
+ && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \
+ && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \
+ && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB))
+# error "No native cmpxchg() op available"
+#endif
+
+
/*
- * --- Pointer size atomics ---------------------------------------------------
+ * Read op used together with cmpxchg() fallback when no native op present.
*/
+#if defined(ETHR_HAVE_NATMC_READ)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+ ((ETHR_NAINT_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+ ethr_sint_t AVAL; \
+ ETHR_NAINT_T__ new__, act__, exp__; \
+ act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
+ do { \
+ exp__ = act__; \
+ AVAL = (ethr_sint_t) act__; \
+ { OPS; } \
+ new__ = (ETHR_NAINT_T__) AVAL; \
+ act__ = CMPXCHG(VAR, new__, exp__); \
+ } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
+
+
+
+/* --- addr() --- */
-static ETHR_INLINE ethr_sint_t *
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var);
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#else
- return (ethr_sint_t *) var;
+#error "Missing implementation of ethr_atomic_cmpxchg()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#error "Missing implementation of ethr_atomic_cmpxchg_rb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#error "Missing implementation of ethr_atomic_cmpxchg_wb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
+#error "Missing implementation of ethr_atomic_cmpxchg_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_relb()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_mb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
+
+/* --- xchg() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#error "Missing implementation of ethr_atomic_xchg()!"
#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
+#error "Missing implementation of ethr_atomic_xchg_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+#if defined(ETHR_HAVE_NATMC_XCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic_xchg_wb()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_xchg_acqb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(inc)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#error "Missing implementation of ethr_atomic_xchg_relb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(dec)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_xchg_mb()!"
+#endif
+ return res;
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET)
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET)
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_MB)
+ ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+ ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val);
+#endif
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#else
+ ETHR_ATMC_FUNC__(set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#else
+ ETHR_ATMC_FUNC__(set_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATMC_FUNC__(set_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+ ETHR_ATMC_FUNC__(set_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_MB)
+ ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+ ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATMC_FUNC__(set_mb)(var, val);
+#endif
+}
+
+
+/* --- add_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic_add_read()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_add_read_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic_add_read_wb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_add_read_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#error "Missing implementation of ethr_atomic_add_read_relb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
+#error "Missing implementation of ethr_atomic_add_read_mb()!"
+#endif
+ return res;
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#if defined(ETHR_HAVE_NATMC_READ)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
#else
+ res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#if defined(ETHR_HAVE_NATMC_READ_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+ res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
- ethr_sint_t mask)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var,
- (ETHR_NAINT_T__) mask);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
+ res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+ return res;
+}
+
+
+/* --- inc_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+#if defined(ETHR_HAVE_NATMC_INC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
- ethr_sint_t mask)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var,
- (ETHR_NAINT_T__) mask);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
#else
+ res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
+
+/* --- dec_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var,
- (ETHR_NAINT_T__) new);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#else
+ res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var,
- (ETHR_NAINT_T__) new,
- (ETHR_NAINT_T__) exp);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
+ res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var)
+{
ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
- {
- res = *var;
- if (__builtin_expect(res == exp, 1))
- *var = new;
- });
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1);
#endif
+ return res;
}
-/*
- * Important memory barrier requirements.
- *
- * The following atomic operations *must* supply a memory barrier of
- * at least the type specified by its suffix:
- * _acqb = acquire barrier
- * _relb = release barrier
- */
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
+/* --- add() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#if defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
+ (void) ETHR_ATMC_FUNC__(add_read)(var, val);
#endif
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#if defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
+ (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val);
#endif
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
- ethr_sint_t val)
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#if defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
+ (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val);
#endif
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
+#if defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+ (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_MB)
+ ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+ ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val);
+#endif
+}
+
+
+/* --- inc() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC)
+ ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+ ETHR_NATMC_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC)
+ ETHR_NATMC_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_MB)
+ ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+ ETHR_NATMC_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(inc_read_mb)(var);
+#endif
+}
+
+
+/* --- dec() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+ ETHR_NATMC_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_NATMC_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+ ETHR_NATMC_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_FUNC__(dec_relb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_NATMC_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+ ETHR_NATMC_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_RELB)
+ ETHR_NATMC_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_MB)
+ ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
ETHR_NATMC_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC_FUNC__(dec_read_mb)(var);
+#endif
+}
+
+
+/* --- read_band() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic_read_band()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_read_band_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
+#error "Missing implementation of ethr_atomic_read_band_wb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
+#error "Missing implementation of ethr_atomic_read_band_acqb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var,
- (ETHR_NAINT_T__) new,
- (ETHR_NAINT_T__) exp);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#error "Missing implementation of ethr_atomic_read_band_relb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var,
- (ETHR_NAINT_T__) new,
- (ETHR_NAINT_T__) exp);
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#error "Missing implementation of ethr_atomic_read_band_mb()!"
#endif
+ return res;
}
+
+/* --- read_bor() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_wb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_relb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_mb()!"
+#endif
+ return res;
+}
+
+#endif /* ETHR_ATMC_INLINE__ */
+
+
+/* ---------- 32-bit atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_ATMC32_PROTOTYPES__
+ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var);
+void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_inc(ethr_atomic32_t *var);
+void ethr_atomic32_inc_rb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_wb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_acqb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_relb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_mb(ethr_atomic32_t *var);
+void ethr_atomic32_dec(ethr_atomic32_t *var);
+void ethr_atomic32_dec_rb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_wb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_acqb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_relb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+#endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \
+ && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_NATMC32_BITS__)
+# error "Missing native atomic implementation"
+#elif ETHR_NATMC32_BITS__ == 64
+# undef ETHR_HAVE_NATMC32_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+# define ETHR_HAVE_NATMC32_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+# define ETHR_HAVE_NATMC32_XCHG 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+# define ETHR_HAVE_NATMC32_XCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+# define ETHR_HAVE_NATMC32_XCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+# define ETHR_HAVE_NATMC32_XCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+# define ETHR_HAVE_NATMC32_XCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+# define ETHR_HAVE_NATMC32_XCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+# define ETHR_HAVE_NATMC32_SET 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+# define ETHR_HAVE_NATMC32_SET_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+# define ETHR_HAVE_NATMC32_SET_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+# define ETHR_HAVE_NATMC32_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+# define ETHR_HAVE_NATMC32_SET_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+# define ETHR_HAVE_NATMC32_SET_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+# define ETHR_HAVE_NATMC32_INIT 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+# define ETHR_HAVE_NATMC32_INIT_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+# define ETHR_HAVE_NATMC32_INIT_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+# define ETHR_HAVE_NATMC32_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+# define ETHR_HAVE_NATMC32_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+# define ETHR_HAVE_NATMC32_INIT_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+# define ETHR_HAVE_NATMC32_ADD_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+# define ETHR_HAVE_NATMC32_READ 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+# define ETHR_HAVE_NATMC32_READ_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+# define ETHR_HAVE_NATMC32_READ_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+# define ETHR_HAVE_NATMC32_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+# define ETHR_HAVE_NATMC32_READ_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+# define ETHR_HAVE_NATMC32_READ_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+# define ETHR_HAVE_NATMC32_INC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+# define ETHR_HAVE_NATMC32_DEC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+# define ETHR_HAVE_NATMC32_ADD 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+# define ETHR_HAVE_NATMC32_ADD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+# define ETHR_HAVE_NATMC32_ADD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+# define ETHR_HAVE_NATMC32_ADD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+# define ETHR_HAVE_NATMC32_ADD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+# define ETHR_HAVE_NATMC32_ADD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+# define ETHR_HAVE_NATMC32_INC 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+# define ETHR_HAVE_NATMC32_INC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+# define ETHR_HAVE_NATMC32_INC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+# define ETHR_HAVE_NATMC32_INC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+# define ETHR_HAVE_NATMC32_INC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+# define ETHR_HAVE_NATMC32_INC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+# define ETHR_HAVE_NATMC32_DEC 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+# define ETHR_HAVE_NATMC32_DEC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+# define ETHR_HAVE_NATMC32_DEC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+# define ETHR_HAVE_NATMC32_DEC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+# define ETHR_HAVE_NATMC32_DEC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+# define ETHR_HAVE_NATMC32_DEC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+# define ETHR_HAVE_NATMC32_AND_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+# define ETHR_HAVE_NATMC32_OR_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
+# endif
+#elif ETHR_NATMC32_BITS__ == 32
+# undef ETHR_HAVE_NATMC32_CMPXCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+# define ETHR_HAVE_NATMC32_CMPXCHG 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+# define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+# define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+# define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+# define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_CMPXCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+# define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+# define ETHR_HAVE_NATMC32_XCHG 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+# define ETHR_HAVE_NATMC32_XCHG_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+# define ETHR_HAVE_NATMC32_XCHG_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+# define ETHR_HAVE_NATMC32_XCHG_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+# define ETHR_HAVE_NATMC32_XCHG_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_XCHG_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+# define ETHR_HAVE_NATMC32_XCHG_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+# define ETHR_HAVE_NATMC32_SET 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+# define ETHR_HAVE_NATMC32_SET_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+# define ETHR_HAVE_NATMC32_SET_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+# define ETHR_HAVE_NATMC32_SET_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+# define ETHR_HAVE_NATMC32_SET_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_SET_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+# define ETHR_HAVE_NATMC32_SET_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+# define ETHR_HAVE_NATMC32_INIT 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+# define ETHR_HAVE_NATMC32_INIT_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+# define ETHR_HAVE_NATMC32_INIT_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+# define ETHR_HAVE_NATMC32_INIT_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+# define ETHR_HAVE_NATMC32_INIT_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INIT_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+# define ETHR_HAVE_NATMC32_INIT_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+# define ETHR_HAVE_NATMC32_ADD_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+# define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+# define ETHR_HAVE_NATMC32_READ 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+# define ETHR_HAVE_NATMC32_READ_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+# define ETHR_HAVE_NATMC32_READ_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+# define ETHR_HAVE_NATMC32_READ_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+# define ETHR_HAVE_NATMC32_READ_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_READ_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+# define ETHR_HAVE_NATMC32_READ_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+# define ETHR_HAVE_NATMC32_INC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+# define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+# define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+# define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+# define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+# define ETHR_HAVE_NATMC32_DEC_RETURN 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+# define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+# define ETHR_HAVE_NATMC32_ADD 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+# define ETHR_HAVE_NATMC32_ADD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+# define ETHR_HAVE_NATMC32_ADD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+# define ETHR_HAVE_NATMC32_ADD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+# define ETHR_HAVE_NATMC32_ADD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_ADD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+# define ETHR_HAVE_NATMC32_ADD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+# define ETHR_HAVE_NATMC32_INC 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+# define ETHR_HAVE_NATMC32_INC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+# define ETHR_HAVE_NATMC32_INC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+# define ETHR_HAVE_NATMC32_INC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+# define ETHR_HAVE_NATMC32_INC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_INC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+# define ETHR_HAVE_NATMC32_INC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+# define ETHR_HAVE_NATMC32_DEC 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+# define ETHR_HAVE_NATMC32_DEC_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+# define ETHR_HAVE_NATMC32_DEC_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+# define ETHR_HAVE_NATMC32_DEC_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+# define ETHR_HAVE_NATMC32_DEC_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_DEC_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+# define ETHR_HAVE_NATMC32_DEC_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+# define ETHR_HAVE_NATMC32_AND_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+# define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+# define ETHR_HAVE_NATMC32_OR_RETOLD 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
+# endif
+# undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
+# ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+# define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
+# endif
+#else
+# error "Invalid native atomic size"
+#endif
+
+#if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \
+ && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \
+ && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \
+ && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \
+ && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \
+ && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB))
+# error "No native cmpxchg() op available"
+#endif
+
+
/*
- * --- 32-bit atomics ---------------------------------------------------------
+ * Read op used together with cmpxchg() fallback when no native op present.
*/
+#if defined(ETHR_HAVE_NATMC32_READ)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ETHR_NATMC32_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+ ((ETHR_NAINT32_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+ ethr_sint32_t AVAL; \
+ ETHR_NAINT32_T__ new__, act__, exp__; \
+ act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \
+ do { \
+ exp__ = act__; \
+ AVAL = (ethr_sint32_t) act__; \
+ { OPS; } \
+ new__ = (ETHR_NAINT32_T__) AVAL; \
+ act__ = CMPXCHG(VAR, new__, exp__); \
+ } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
-static ETHR_INLINE ethr_sint32_t *
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var)
+
+
+/* --- addr() --- */
+
+static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic32_addr(var);
+ return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var);
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#else
- return (ethr_sint32_t *) var;
+#error "Missing implementation of ethr_atomic32_cmpxchg()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var,
- ethr_sint32_t i)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#error "Missing implementation of ethr_atomic32_cmpxchg_rb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#error "Missing implementation of ethr_atomic32_cmpxchg_wb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_relb()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_mb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var,
- ethr_sint32_t incr)
+
+/* --- xchg() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#error "Missing implementation of ethr_atomic32_xchg()!"
#endif
-}
-
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var,
- ethr_sint32_t i)
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t)
- ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
+#error "Missing implementation of ethr_atomic32_xchg_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+#if defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_wb()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_acqb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(inc)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#error "Missing implementation of ethr_atomic32_xchg_relb()!"
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(dec)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_mb()!"
+#endif
+ return res;
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_MB)
+ ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+ ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+ ETHR_MEMBAR(ETHR_LoadStore);
+ ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val);
+#endif
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_ATMC32_FUNC__(set)(var, val);
#endif
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var)
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#if defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
+ ETHR_ATMC32_FUNC__(set_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+ ETHR_ATMC32_FUNC__(set_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATMC32_FUNC__(set_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+ ETHR_ATMC32_FUNC__(set_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_MB)
+ ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+ ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATMC32_FUNC__(set_mb)(var, val);
+#endif
+}
+
+
+/* --- add_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic32_add_read()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_rb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#else
+#error "Missing implementation of ethr_atomic32_add_read_wb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_acqb()!"
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_relb()!"
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var,
- ethr_sint32_t mask)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t)
- ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
+#error "Missing implementation of ethr_atomic32_add_read_mb()!"
+#endif
+ return res;
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+#if defined(ETHR_HAVE_NATMC32_READ)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var,
- ethr_sint32_t mask)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return
- (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var,
- (ETHR_NAINT32_T__) mask);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+#if defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var,
- ethr_sint32_t new)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var,
- (ETHR_NAINT32_T__) new);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
+ res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+ return res;
+}
+
+
+/* --- inc_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var,
- (ETHR_NAINT32_T__) new,
- (ETHR_NAINT32_T__) exp);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
#else
+ res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var)
+{
ethr_sint32_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
- {
- res = *var;
- if (__builtin_expect(res == exp, 1))
- *var = new;
- });
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1);
+#endif
return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1);
#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1);
+#endif
+ return res;
}
-/*
- * Important memory barrier requirements.
- *
- * The following atomic operations *must* supply a memory barrier of
- * at least the type specified by its suffix:
- * _acqb = acquire barrier
- * _relb = release barrier
- */
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var)
+/* --- dec_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var);
+ res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1);
#endif
+ return res;
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var);
+ res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1);
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var,
- ethr_sint32_t val)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val);
+ res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1);
#endif
+ return res;
}
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1);
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1);
+#endif
+ return res;
+}
+
+
+/* --- add() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_MB)
+ ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+ ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val);
+#endif
+}
+
+
+/* --- inc() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_NATMC32_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_NATMC32_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_MB)
+ ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+ ETHR_NATMC32_FUNC__(inc_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(inc)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var);
+#endif
+}
+
+
+/* --- dec() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
+#if defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_NATMC32_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#else
+ (void) ETHR_ATMC32_FUNC__(dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_NATMC32_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_NATMC32_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
ETHR_NATMC32_FUNC__(dec_relb)(var);
#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var);
+ (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var);
#endif
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var)
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#if defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_NATMC32_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_NATMC32_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var);
+ (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var);
#endif
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t)
- ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var,
- (ETHR_NAINT32_T__) new,
- (ETHR_NAINT32_T__) exp);
+#if defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_NATMC32_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp);
+ (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var);
#endif
}
-static ETHR_INLINE ethr_sint32_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var)
{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return (ethr_sint32_t)
- ETHR_NATMC32_FUNC__(cmpxchg_relb)(var,
- (ETHR_NAINT32_T__) new,
- (ETHR_NAINT32_T__) exp);
+#if defined(ETHR_HAVE_NATMC32_DEC_MB)
+ ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+ ETHR_NATMC32_FUNC__(dec_relb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_wb)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec_rb)(var);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_FUNC__(dec)(var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp);
+ (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var);
#endif
}
-#endif /* ETHR_TRY_INLINE_FUNCS */
+/* --- read_band() --- */
-#undef ETHR_NAINT_T__
-#undef ETHR_NATMC_FUNC__
-#undef ETHR_NATMC_ADDR_FUNC__
-#undef ETHR_NAINT32_T__
-#undef ETHR_NATMC32_FUNC__
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_rb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_wb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_relb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_mb()!"
+#endif
+ return res;
+}
+
+
+/* --- read_bor() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_rb()!"
+#endif
+ return res;
+}
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_wb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_acqb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_relb()!"
+#endif
+ return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_mb()!"
#endif
+ return res;
+}
+
+#endif /* ETHR_ATMC32_INLINE__ */
+
+#endif /* ETHR_ATOMICS_H__ */
diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h
index e9c3daf783..c9b1db5b46 100644
--- a/erts/include/internal/ethr_internal.h
+++ b/erts/include/internal/ethr_internal.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -63,5 +63,9 @@ int ethr_late_init_common__(ethr_late_init_data *lid);
void ethr_run_exit_handlers__(void);
void ethr_ts_event_destructor__(void *vtsep);
+#if defined(ETHR_X86_RUNTIME_CONF__)
+int ethr_x86_have_cpuid__(void);
+void ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx);
+#endif
#endif
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index fadaf1e2a4..a0685ea3c0 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -54,6 +54,10 @@
# endif
#endif
+#ifndef ETHR_INLINE_MTX_FUNC_NAME_
+# define ETHR_INLINE_MTX_FUNC_NAME_(X) X
+#endif
+
#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
#ifdef ETHR_DEBUG
@@ -505,7 +509,7 @@ void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t);
void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t);
static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
{
ethr_sint32_t act;
int res;
@@ -529,7 +533,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
{
ethr_sint32_t act;
ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
@@ -549,7 +553,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
{
ethr_sint32_t act;
ETHR_COMPILER_BARRIER;
@@ -574,7 +578,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
{
int res;
res = pthread_mutex_trylock(&mtx->pt_mtx);
@@ -584,7 +588,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
{
int res = pthread_mutex_lock(&mtx->pt_mtx);
if (res != 0)
@@ -592,7 +596,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
{
int res = pthread_mutex_unlock(&mtx->pt_mtx);
if (res != 0)
@@ -615,7 +619,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
if (res != 0 && res != EBUSY)
@@ -624,7 +628,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
if (res != 0)
@@ -632,7 +636,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
if (res != 0)
@@ -640,7 +644,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
}
static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
if (res != 0 && res != EBUSY)
@@ -649,7 +653,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
if (res != 0)
@@ -657,7 +661,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
}
static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
{
int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
if (res != 0)
diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h
index 8e04692856..45399d18e6 100644
--- a/erts/include/internal/ethr_optimized_fallbacks.h
+++ b/erts/include/internal/ethr_optimized_fallbacks.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -18,152 +18,172 @@
*/
/*
- * Description: "Optimized" fallbacks used when native ops are missing
+ * Description: Optimized fallbacks used when native ops are missing
* Author: Rickard Green
*/
#ifndef ETHR_OPTIMIZED_FALLBACKS_H__
#define ETHR_OPTIMIZED_FALLBACKS_H__
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
+#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE int
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
+{
+ return 0;
+}
+
#endif
-#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
-#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-/* --- Optimized spinlocks using pthread spinlocks -------------------------- */
-#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
+/* --- Native spinlocks using pthread spinlocks -------------------------- */
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+
+#define ETHR_NATIVE_SPINLOCK_IMPL "pthread"
-typedef pthread_spinlock_t ethr_opt_spinlock_t;
+typedef pthread_spinlock_t ethr_native_spinlock_t;
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
-static ETHR_INLINE int
-ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock)
+static ETHR_INLINE void
+ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
{
- return pthread_spin_init((pthread_spinlock_t *) lock, 0);
+ int err = pthread_spin_init((pthread_spinlock_t *) lock, 0);
+ if (err)
+ ETHR_FATAL_ERROR__(err);
}
static ETHR_INLINE int
-ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock)
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
{
return pthread_spin_destroy((pthread_spinlock_t *) lock);
}
-
-static ETHR_INLINE int
-ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock)
+static ETHR_INLINE void
+ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
- return pthread_spin_unlock((pthread_spinlock_t *) lock);
+ int err = pthread_spin_unlock((pthread_spinlock_t *) lock);
+ if (err)
+ ETHR_FATAL_ERROR__(err);
}
-static ETHR_INLINE int
-ethr_opt_spin_lock(ethr_opt_spinlock_t *lock)
+static ETHR_INLINE void
+ethr_native_spin_lock(ethr_native_spinlock_t *lock)
{
- return pthread_spin_lock((pthread_spinlock_t *) lock);
+ int err = pthread_spin_lock((pthread_spinlock_t *) lock);
+ if (err)
+ ETHR_FATAL_ERROR__(err);
}
#endif
-#elif defined(ETHR_HAVE_NATIVE_ATOMICS)
+#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
/* --- Native spinlocks using native atomics -------------------------------- */
#define ETHR_HAVE_NATIVE_SPINLOCKS 1
-#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1
-
-#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
-typedef ethr_native_atomic32_t ethr_native_spinlock_t;
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
-typedef ethr_native_atomic64_t ethr_native_spinlock_t;
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-#else
-# error "Missing native atomic implementation"
-#endif
+
+#define ETHR_NATIVE_SPINLOCK_IMPL "native-atomics"
+
+typedef ethr_atomic32_t ethr_native_spinlock_t;
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#undef ETHR_NSPN_AOP__
+#define ETHR_NSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
static ETHR_INLINE void
ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
{
- ETHR_NATMC_FUNC__(init)(lock, 0);
+ ETHR_NSPN_AOP__(init)(lock, 0);
+}
+
+static ETHR_INLINE int
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
+{
+ return ETHR_NSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
}
static ETHR_INLINE void
ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
- ETHR_COMPILER_BARRIER;
- ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1);
- ETHR_NATMC_FUNC__(set_relb)(lock, 0);
+ ETHR_ASSERT(ETHR_NSPN_AOP__(read)(lock) == 1);
+ ETHR_NSPN_AOP__(set_relb)(lock, 0);
}
static ETHR_INLINE void
ethr_native_spin_lock(ethr_native_spinlock_t *lock)
{
- while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) {
- while (ETHR_NATMC_FUNC__(read)(lock) != 0)
+ while (ETHR_NSPN_AOP__(cmpxchg_acqb)(lock, 1, 0) != 0) {
+ while (ETHR_NSPN_AOP__(read)(lock) != 0)
ETHR_SPIN_BODY;
}
ETHR_COMPILER_BARRIER;
}
-#endif
+#undef ETHR_NSPN_AOP__
-#undef ETHR_NATMC_FUNC__
+#endif
#endif
-#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
-#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1
-#elif defined(ETHR_HAVE_NATIVE_ATOMICS)
+#if defined(ETHR_HAVE_NATIVE_RWSPINLOCKS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE int
+ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
+{
+ return 0;
+}
+
+#endif
+
+#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
/* --- Native rwspinlocks using native atomics ------------------------------ */
#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
-#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "native-atomics"
-#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
-typedef ethr_native_atomic32_t ethr_native_rwlock_t;
-# define ETHR_NAINT_T__ ethr_sint32_t
+typedef ethr_atomic32_t ethr_native_rwlock_t;
# define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30)
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
-typedef ethr_native_atomic64_t ethr_native_rwlock_t;
-# define ETHR_NAINT_T__ ethr_sint64_t
-# define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62)
-# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-#else
-# error "Missing native atomic implementation"
-#endif
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#undef ETHR_NRWSPN_AOP__
+#define ETHR_NRWSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
static ETHR_INLINE void
ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
{
- ETHR_NATMC_FUNC__(init)(lock, 0);
+ ETHR_NRWSPN_AOP__(init)(lock, 0);
+}
+
+static ETHR_INLINE int
+ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
+{
+ return ETHR_NRWSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
}
static ETHR_INLINE void
ethr_native_read_unlock(ethr_native_rwlock_t *lock)
{
- ETHR_COMPILER_BARRIER;
-#ifdef DEBUG
- ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0);
-#endif
- ETHR_NATMC_FUNC__(dec_relb)(lock);
+ ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) >= 0);
+ ETHR_NRWSPN_AOP__(dec_relb)(lock);
}
static ETHR_INLINE void
ethr_native_read_lock(ethr_native_rwlock_t *lock)
{
- ETHR_NAINT_T__ act, exp = 0;
+ ethr_sint32_t act, exp = 0;
while (1) {
- act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp);
+ act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp+1, exp);
if (act == exp)
break;
+ /* Wait for writer to leave */
while (act & ETHR_WLOCK_FLAG__) {
ETHR_SPIN_BODY;
- act = ETHR_NATMC_FUNC__(read)(lock);
+ act = ETHR_NRWSPN_AOP__(read)(lock);
}
exp = act;
}
@@ -173,36 +193,37 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
static ETHR_INLINE void
ethr_native_write_unlock(ethr_native_rwlock_t *lock)
{
- ETHR_COMPILER_BARRIER;
- ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__);
- ETHR_NATMC_FUNC__(set_relb)(lock, 0);
+ ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) == ETHR_WLOCK_FLAG__);
+ ETHR_NRWSPN_AOP__(set_relb)(lock, 0);
}
static ETHR_INLINE void
ethr_native_write_lock(ethr_native_rwlock_t *lock)
{
- ETHR_NAINT_T__ act, exp = 0;
+ ethr_sint32_t act, exp = 0;
while (1) {
- act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp);
+ act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp);
if (act == exp)
break;
- ETHR_SPIN_BODY;
- exp = act & ~ETHR_WLOCK_FLAG__;
+ /* Wait for writer to leave */
+ while (act & ETHR_WLOCK_FLAG__) {
+ ETHR_SPIN_BODY;
+ act = ETHR_NRWSPN_AOP__(read)(lock);
+ }
+ exp = act;
}
act |= ETHR_WLOCK_FLAG__;
/* Wait for readers to leave */
while (act != ETHR_WLOCK_FLAG__) {
ETHR_SPIN_BODY;
- act = ETHR_NATMC_FUNC__(read_acqb)(lock);
+ act = ETHR_NRWSPN_AOP__(read_acqb)(lock);
}
ETHR_COMPILER_BARRIER;
}
-#endif
+#undef ETHR_NRWSPN_AOP__
-#undef ETHR_NAINT_T__
-#undef ETHR_NATMC_FUNC__
-#undef ETHR_WLOCK_FLAG__
+#endif
#endif
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 4cd95faf6a..8ad0ded144 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -33,10 +33,6 @@
#include <stdlib.h>
#include "erl_errno.h"
-#undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-#undef ETHR_HAVE_OPTIMIZED_SPINLOCK
-#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK
-
#if defined(DEBUG)
# define ETHR_DEBUG
#endif
@@ -68,7 +64,7 @@
#endif
/* Assume 64-byte cache line size */
-#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64)
+#define ETHR_CACHE_LINE_SIZE 64
#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1)
#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \
@@ -251,13 +247,90 @@ typedef ethr_sint64_t ethr_sint_t;
typedef ethr_uint64_t ethr_uint_t;
#endif
-/* __builtin_expect() is needed by both native atomics code
- * and the fallback code */
-#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
+#if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16
+#define ETHR_HAVE_INT128_T
+typedef __int128_t ethr_sint128_t;
+typedef __uint128_t ethr_uint128_t;
+#endif
+
+#define ETHR_FATAL_ERROR__(ERR) \
+ ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))
+
+ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
+ int line,
+ const char *func,
+ int err);
+
+#if !defined(__GNUC__)
+# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+# define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
+#if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0)
#define __builtin_expect(X, Y) (X)
#endif
-/* For CPU-optimised atomics, spinlocks, and rwlocks. */
+#if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1)
+# define ETHR_CHOOSE_EXPR __builtin_choose_expr
+#else
+# define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2))
+#endif
+
+#if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \
+ || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))))
+# define ETHR_X86_RUNTIME_CONF__
+
+# define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \
+ (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg != 0, 1))
+# define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \
+ (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg == 0, 0))
+# define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \
+ (__builtin_expect(ethr_runtime__.conf.have_sse2 != 0, 1))
+# define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \
+ (__builtin_expect(ethr_runtime__.conf.have_sse2 == 0, 0))
+#endif
+
+#if (defined(__GNUC__) \
+ && !defined(ETHR_PPC_HAVE_LWSYNC) \
+ && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \
+ && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)))
+# define ETHR_PPC_RUNTIME_CONF__
+
+# define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \
+ (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1))
+# define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \
+ (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0))
+#endif
+
+typedef struct {
+#if defined(ETHR_X86_RUNTIME_CONF__)
+ int have_dw_cmpxchg;
+ int have_sse2;
+#endif
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+ int have_lwsync;
+#endif
+ int dummy;
+} ethr_runtime_conf_t;
+
+
+typedef union {
+ ethr_runtime_conf_t conf;
+ char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE];
+} ethr_runtime_t;
+
+
+extern ethr_runtime_t ethr_runtime__;
+
+/* For native CPU-optimised atomics, spinlocks, and rwlocks. */
#if !defined(ETHR_DISABLE_NATIVE_IMPLS)
# if defined(__GNUC__)
# if defined(ETHR_PREFER_GCC_NATIVE_IMPLS)
@@ -265,7 +338,7 @@ typedef ethr_uint64_t ethr_uint_t;
# elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
# include "libatomic_ops/ethread.h"
# endif
-# ifndef ETHR_HAVE_NATIVE_ATOMICS
+# if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
# if ETHR_SIZEOF_PTR == 4
# if defined(__i386__)
# include "i386/ethread.h"
@@ -283,8 +356,10 @@ typedef ethr_uint64_t ethr_uint_t;
# include "sparc64/ethread.h"
# endif
# endif
+#if 0
# include "gcc/ethread.h"
# include "libatomic_ops/ethread.h"
+#endif
# endif
# elif defined(ETHR_HAVE_LIBATOMIC_OPS)
# include "libatomic_ops/ethread.h"
@@ -293,10 +368,9 @@ typedef ethr_uint64_t ethr_uint_t;
# endif
#endif /* !ETHR_DISABLE_NATIVE_IMPLS */
+#include "ethr_atomics.h" /* The atomics API */
+
#if defined(__GNUC__)
-# ifndef ETHR_COMPILER_BARRIER
-# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
-# endif
# ifndef ETHR_SPIN_BODY
# if defined(__i386__) || defined(__x86_64__)
# define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory")
@@ -309,11 +383,6 @@ typedef ethr_uint64_t ethr_uint_t;
# endif
# endif
#elif defined(ETHR_WIN32_THREADS)
-# ifndef ETHR_COMPILER_BARRIER
-# include <intrin.h>
-# pragma intrinsic(_ReadWriteBarrier)
-# define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
-# endif
# ifndef ETHR_SPIN_BODY
# define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0)
# endif
@@ -321,43 +390,6 @@ typedef ethr_uint64_t ethr_uint_t;
#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
- * i.e. when our lock based atomic fallback is used, a noop is sufficient.
- */
-#define ETHR_MEMORY_BARRIER do { } while (0)
-#define ETHR_WRITE_MEMORY_BARRIER do { } while (0)
-#define ETHR_READ_MEMORY_BARRIER do { } while (0)
-#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0)
-#endif
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER
-# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL
-#endif
-#ifndef ETHR_READ_MEMORY_BARRIER
-# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-# define ETHR_READ_MEMORY_BARRIER_IS_FULL
-#endif
-#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER
-# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
-# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER
-#endif
-
-#define ETHR_FATAL_ERROR__(ERR) \
- ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))
-
-ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
- int line,
- const char *func,
- int err);
-
-void ethr_compiler_barrier_fallback(void);
-#ifndef ETHR_COMPILER_BARRIER
-# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
-#endif
-
#ifndef ETHR_SPIN_BODY
# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
#endif
@@ -460,8 +492,6 @@ void ethr_compiler_barrier(void);
#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
typedef ethr_native_spinlock_t ethr_spinlock_t;
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
-typedef ethr_opt_spinlock_t ethr_spinlock_t;
#elif defined(__WIN32__)
typedef CRITICAL_SECTION ethr_spinlock_t;
#else
@@ -483,8 +513,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
ethr_native_spinlock_init(lock);
return 0;
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
- return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock);
#elif defined(__WIN32__)
if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX))
return ethr_win_get_errno__();
@@ -498,9 +526,7 @@ static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
- return 0;
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
- return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock);
+ return ethr_native_spinlock_destroy(lock);
#elif defined(__WIN32__)
DeleteCriticalSection((CRITICAL_SECTION *) lock);
return 0;
@@ -514,10 +540,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
ethr_native_spin_unlock(lock);
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
- int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock);
- if (err)
- ETHR_FATAL_ERROR__(err);
#elif defined(__WIN32__)
LeaveCriticalSection((CRITICAL_SECTION *) lock);
#else
@@ -532,10 +554,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
ethr_native_spin_lock(lock);
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS)
- int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock);
- if (err)
- ETHR_FATAL_ERROR__(err);
#elif defined(__WIN32__)
EnterCriticalSection((CRITICAL_SECTION *) lock);
#else
@@ -547,8 +565,6 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
#endif /* ETHR_TRY_INLINE_FUNCS */
-#include "ethr_atomics.h"
-
typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
#if defined(ETHR_WIN32_THREADS)
diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in
index f394d790d2..dd3599f86d 100644
--- a/erts/include/internal/ethread_header_config.h.in
+++ b/erts/include/internal/ethread_header_config.h.in
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -32,6 +32,9 @@
/* Define to the size of __int64 */
#undef ETHR_SIZEOF___INT64
+/* Define to the size of __int128_t */
+#undef ETHR_SIZEOF___INT128_T
+
/* Define if bigendian */
#undef ETHR_BIGENDIAN
@@ -69,8 +72,44 @@
/* Define if you have a linux futex implementation. */
#undef ETHR_HAVE_LINUX_FUTEX
-/* Define if you have gcc atomic operations */
-#undef ETHR_HAVE_GCC_ATOMIC_OPS
+/* Define if x86/x86_64 out of order instructions should be synchronized */
+#undef ETHR_X86_OUT_OF_ORDER
+
+/* Define if only run in Sparc TSO mode */
+#undef ETHR_SPARC_TSO
+
+/* Define if only run in Sparc PSO, or TSO mode */
+#undef ETHR_SPARC_PSO
+
+/* Define if run in Sparc RMO, PSO, or TSO mode */
+#undef ETHR_SPARC_RMO
+
+/* Define if you have __sync_add_and_fetch() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH32
+
+/* Define if you have __sync_add_and_fetch() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH64
+
+/* Define if you have __sync_fetch_and_and() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND32
+
+/* Define if you have __sync_fetch_and_and() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND64
+
+/* Define if you have __sync_fetch_and_or() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR32
+
+/* Define if you have __sync_fetch_and_or() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR64
+
+/* Define if you have __sync_val_compare_and_swap() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32
+
+/* Define if you have __sync_val_compare_and_swap() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64
+
+/* Define if you have __sync_val_compare_and_swap() for 128-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128
/* Define if you prefer gcc native ethread implementations */
#undef ETHR_PREFER_GCC_NATIVE_IMPLS
@@ -90,8 +129,14 @@
/* Define if sched_yield() returns an int. */
#undef ETHR_SCHED_YIELD_RET_INT
-/* Define if you want compatibilty with x86 processors before pentium4. */
-#undef ETHR_PRE_PENTIUM4_COMPAT
+/* Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements */
+#undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT
+
+/* Define if you use a gcc that supports the double word cmpxchg instruction */
+#undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT
+
+/* Define if you get a register shortage with cmpxchg8b and position independent code */
+#undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE
/* Define if you have the pthread_rwlockattr_setkind_np() function. */
#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP
@@ -115,23 +160,74 @@
/* Define to the size of AO_t if libatomic_ops is used */
#undef ETHR_SIZEOF_AO_T
+/* Define if you have _InterlockedAnd() */
+#undef ETHR_HAVE__INTERLOCKEDAND
+
+/* Define if you have _InterlockedAnd64() */
+#undef ETHR_HAVE__INTERLOCKEDAND64
+
+/* Define if you have _InterlockedCompareExchange() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
+
/* Define if you have _InterlockedCompareExchange64() */
#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+/* Define if you have _InterlockedCompareExchange64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
+
+/* Define if you have _InterlockedCompareExchange64_rel() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
+
+/* Define if you have _InterlockedCompareExchange_acq() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
+
+/* Define if you have _InterlockedCompareExchange_rel() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
+
+/* Define if you have _InterlockedDecrement() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT
+
/* Define if you have _InterlockedDecrement64() */
#undef ETHR_HAVE__INTERLOCKEDDECREMENT64
-/* Define if you have _InterlockedIncrement64() */
-#undef ETHR_HAVE__INTERLOCKEDINCREMENT64
+/* Define if you have _InterlockedDecrement64_rel() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
-/* Define if you have _InterlockedExchangeAdd64() */
-#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+/* Define if you have _InterlockedDecrement_rel() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
+
+/* Define if you have _InterlockedExchange() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGE
/* Define if you have _InterlockedExchange64() */
#undef ETHR_HAVE__INTERLOCKEDEXCHANGE64
-/* Define if you have _InterlockedAnd64() */
-#undef ETHR_HAVE__INTERLOCKEDAND64
+/* Define if you have _InterlockedExchangeAdd() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
+
+/* Define if you have _InterlockedExchangeAdd64() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+
+/* Define if you have _InterlockedExchangeAdd64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
+
+/* Define if you have _InterlockedExchangeAdd_acq() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
+
+/* Define if you have _InterlockedIncrement() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT
+
+/* Define if you have _InterlockedIncrement64() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT64
+
+/* Define if you have _InterlockedIncrement64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
+
+/* Define if you have _InterlockedIncrement_acq() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
+
+/* Define if you have _InterlockedOr() */
+#undef ETHR_HAVE__INTERLOCKEDOR
/* Define if you have _InterlockedOr64() */
#undef ETHR_HAVE__INTERLOCKEDOR64
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
index 16935084b1..f598f8537b 100644
--- a/erts/include/internal/gcc/ethr_atomic.h
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,11 +25,15 @@
#undef ETHR_INCLUDE_ATOMIC_IMPL__
#if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
#define ETHR_GCC_ATOMIC32_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32)
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#endif
#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
#elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
#define ETHR_GCC_ATOMIC64_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#endif
#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
#endif
@@ -45,58 +49,38 @@
# define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1
#endif
-#if defined(__x86_64__) || (defined(__i386__) \
- && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1
-#else
-# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0
-#endif
-
-/*
- * According to the documentation this is what we want:
- * #define ETHR_MEMORY_BARRIER __sync_synchronize()
- * However, __sync_synchronize() is known to erroneously be
- * a noop on at least some platforms with some gcc versions.
- * This has suposedly been fixed in some gcc version, but we
- * don't know from which version. Therefore, we only use
- * it when it has been verified to work. Otherwise
- * we use a workaround.
- */
-#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2))
-/* __sync_synchronize() has been verified to work here */
-#define ETHR_MEMORY_BARRIER __sync_synchronize()
-#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize()
-#elif defined(__x86_64__) || (defined(__i386__) \
- && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-/* Use fence instructions directly instead of workaround */
-#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory")
-#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory")
-#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory")
-#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory")
-#else
-/* Workaround */
-#define ETHR_MEMORY_BARRIER \
-do { \
- volatile ethr_sint32_t x___ = 0; \
- (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \
-} while (0)
-#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-
-#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
-
#endif /* ETHR_GCC_ATOMIC_COMMON__ */
#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "gcc"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic32_t
#define ETHR_AINT_T__ ethr_sint32_t
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH32)
+# define ETHR_HAVE___SYNC_ADD_AND_FETCH
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND32)
+# define ETHR_HAVE___SYNC_FETCH_AND_AND
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR32)
+# define ETHR_HAVE___SYNC_FETCH_AND_OR
+#endif
#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "gcc"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic64_t
#define ETHR_AINT_T__ ethr_sint64_t
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH64)
+# define ETHR_HAVE___SYNC_ADD_AND_FETCH
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND64)
+# define ETHR_HAVE___SYNC_FETCH_AND_AND
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR64)
+# define ETHR_HAVE___SYNC_FETCH_AND_OR
+#endif
#else
#error "Unsupported integer size"
#endif
@@ -108,183 +92,120 @@ typedef struct {
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
{
return (ETHR_AINT_T__ *) &var->counter;
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
-{
#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
- var->counter = value;
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
#else
- /*
- * Unfortunately no __sync_store() or similar exist in the gcc atomic
- * op interface. We therefore have to simulate it this way...
- */
- ETHR_AINT_T__ act = 0, exp;
- do {
- exp = act;
- act = __sync_val_compare_and_swap(&var->counter, exp, value);
- } while (act != exp);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
#endif
-}
static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
- ETHR_NATMC_FUNC__(set)(var, value);
+ var->counter = value;
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
-{
+#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */
+
#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
- return var->counter;
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
#else
- /*
- * Unfortunately no __sync_fetch() or similar exist in the gcc atomic
- * op interface. We therefore have to simulate it this way...
- */
- return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
#endif
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- (void) __sync_add_and_fetch(&var->counter, incr);
-}
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
- return __sync_add_and_fetch(&var->counter, incr);
+ return var->counter;
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
- (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
-}
+#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
- (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
-}
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH)
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
- return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
- return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1);
+ return __sync_add_and_fetch(&var->counter, incr);
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- return __sync_fetch_and_and(&var->counter, mask);
-}
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask);
-}
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND)
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
-{
- return __sync_val_compare_and_swap(&var->counter, old, new);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
- ETHR_AINT_T__ exp, act = 0;
- do {
- exp = act;
- act = __sync_val_compare_and_swap(&var->counter, exp, new);
- } while (act != exp);
- return act;
+ return __sync_fetch_and_and(&var->counter, mask);
}
-/*
- * Atomic ops with at least specified barriers.
- */
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
- ETHR_AINT_T__ val = var->counter;
- ETHR_COMPILER_BARRIER;
- return val;
-#else
- return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0);
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
- ETHR_COMPILER_BARRIER;
- var->counter = i;
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
#else
- (void) ETHR_NATMC_FUNC__(xchg)(var, i);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(inc_return)(var);
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
- ETHR_NATMC_FUNC__(dec)(var);
+ return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask);
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(dec_return)(var);
-}
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
-{
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+ ETHR_AINT_T__ new,
+ ETHR_AINT_T__ old)
{
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+ return __sync_val_compare_and_swap(&var->counter, old, new);
}
-#endif
+#endif /* ETHR_TRY_INLINE_FUNCS */
#undef ETHR_NATMC_FUNC__
#undef ETHR_ATMC_T__
#undef ETHR_AINT_T__
#undef ETHR_AINT_SUFFIX__
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR
#endif
diff --git a/erts/include/internal/gcc/ethr_dw_atomic.h b/erts/include/internal/gcc/ethr_dw_atomic.h
new file mode 100644
index 0000000000..6736f9c547
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_dw_atomic.h
@@ -0,0 +1,115 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#ifndef ETHR_GCC_DW_ATOMIC_H__
+# define ETHR_GCC_DW_ATOMIC_H__
+# if ((ETHR_SIZEOF_PTR == 4 \
+ && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)) \
+ || (ETHR_SIZEOF_PTR == 8 \
+ && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128) \
+ && defined(ETHR_HAVE_INT128_T)))
+# define ETHR_INCLUDE_DW_ATOMIC_IMPL__
+# endif
+#endif
+
+#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+# define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc"
+
+# if defined(__i386__) || defined(__x86_64__)
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+ ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+# endif
+
+# if ETHR_SIZEOF_PTR == 4
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+# elif ETHR_SIZEOF_PTR == 8
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t
+# endif
+
+typedef volatile ETHR_NATIVE_SU_DW_SINT_T * ethr_native_dw_ptr_t;
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+ (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+ volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+ volatile ethr_sint_t sint[3];
+ volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+# ifdef ETHR_DEBUG
+# define ETHR_DW_DBG_ALIGNED__(PTR) \
+ ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+# else
+# define ETHR_DW_DBG_ALIGNED__(PTR)
+# endif
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+ return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
+ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+ ETHR_NATIVE_SU_DW_SINT_T new,
+ ETHR_NATIVE_SU_DW_SINT_T old)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ ETHR_DW_DBG_ALIGNED__(p);
+ return __sync_val_compare_and_swap(p, old, new);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHR_GCC_DW_ATOMIC_H__ */
+
diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h
new file mode 100644
index 0000000000..7d428fc68e
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_membar.h
@@ -0,0 +1,73 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers when using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_GCC_MEMBAR_H__
+#define ETHR_GCC_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+/*
+ * According to the documentation __sync_synchronize() will
+ * issue a full memory barrier. However, __sync_synchronize()
+ * is known to erroneously be a noop on at least some
+ * platforms with some gcc versions. This has suposedly been
+ * fixed in some gcc version, but we don't know from which
+ * version. Therefore, we only use it when it has been
+ * verified to work. Otherwise we use the workaround
+ * below.
+ */
+
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32)
+# define ETHR_MB_T__ ethr_sint32_t
+#elif defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)
+# define ETHR_MB_T__ ethr_sint64_t
+#else
+# error "No __sync_val_compare_and_swap"
+#endif
+#define ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ \
+do { \
+ volatile ETHR_MB_T__ x___ = 0; \
+ (void) __sync_val_compare_and_swap(&x___, (ETHR_MB_T__) 0, (ETHR_MB_T__) 1); \
+} while (0)
+
+#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+
+#if defined(__mips__) && ETHR_AT_LEAST_GCC_VSN__(4, 2, 0)
+# define ETHR_MEMBAR(B) __sync_synchronize()
+# define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize()
+#elif ((defined(__powerpc__) || defined(__ppc__)) \
+ && ETHR_AT_LEAST_GCC_VSN__(4, 1, 2))
+# define ETHR_MEMBAR(B) __sync_synchronize()
+#else /* Use workaround */
+# define ETHR_MEMBAR(B) \
+ ETHR_SYNC_SYNCHRONIZE_WORKAROUND__
+# define ETHR_READ_DEPEND_MEMORY_BARRIER \
+ ETHR_SYNC_SYNCHRONIZE_WORKAROUND__
+#endif
+
+
+#endif /* ETHR_GCC_MEMBAR_H__ */
diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h
index 392a1aa2b2..fcfdc39441 100644
--- a/erts/include/internal/gcc/ethread.h
+++ b/erts/include/internal/gcc/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,16 +25,24 @@
#ifndef ETHREAD_GCC_H__
#define ETHREAD_GCC_H__
-#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS)
-#define ETHR_HAVE_NATIVE_ATOMICS 1
+#ifndef ETHR_MEMBAR
+# include "ethr_membar.h"
+#endif
+
+#if !defined(ETHR_HAVE_NATIVE_ATOMIC32)
+# define ETHR_ATOMIC_WANT_32BIT_IMPL__
+# include "ethr_atomic.h"
+#endif
-#define ETHR_ATOMIC_WANT_32BIT_IMPL__
-#include "ethr_atomic.h"
-#if ETHR_SIZEOF_PTR == 8
+#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
# define ETHR_ATOMIC_WANT_64BIT_IMPL__
# include "ethr_atomic.h"
#endif
+#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \
+ && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \
+ && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128)))
+# include "ethr_dw_atomic.h"
#endif
#endif
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index 4e402f261a..fc1b619935 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,53 +25,42 @@
*/
#undef ETHR_INCLUDE_ATOMIC_IMPL__
-#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
-#define ETHR_X86_ATOMIC32_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
-#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
-#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
-#define ETHR_X86_ATOMIC64_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
-#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#if !defined(ETHR_X86_ATOMIC32_H__) \
+ && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+# define ETHR_X86_ATOMIC32_H__
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+# undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_X86_ATOMIC64_H__) \
+ && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+# define ETHR_X86_ATOMIC64_H__
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+# undef ETHR_ATOMIC_WANT_64BIT_IMPL__
#endif
#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
-#ifndef ETHR_X86_ATOMIC_COMMON__
-#define ETHR_X86_ATOMIC_COMMON__
-
-#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
-
-#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
-#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory")
-#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory")
-#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory")
-#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory")
-#else
-#define ETHR_MEMORY_BARRIER \
-do { \
- volatile ethr_sint32_t x___ = 0; \
- __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \
-} while (0)
-#endif
-
-#endif /* ETHR_X86_ATOMIC_COMMON__ */
-
-#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
-#define ETHR_HAVE_NATIVE_ATOMIC32 1
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic32_t
-#define ETHR_AINT_T__ ethr_sint32_t
-#define ETHR_AINT_SUFFIX__ "l"
-#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
-#define ETHR_HAVE_NATIVE_ATOMIC64 1
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic64_t
-#define ETHR_AINT_T__ ethr_sint64_t
-#define ETHR_AINT_SUFFIX__ "q"
-#else
-#error "Unsupported integer size"
-#endif
+# ifndef ETHR_X86_ATOMIC_COMMON__
+# define ETHR_X86_ATOMIC_COMMON__
+# define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
+# endif /* ETHR_X86_ATOMIC_COMMON__ */
+
+# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_NATIVE_ATOMIC32 1
+# define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_ATMC_T__ ethr_native_atomic32_t
+# define ETHR_AINT_T__ ethr_sint32_t
+# define ETHR_AINT_SUFFIX__ "l"
+# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+# define ETHR_HAVE_NATIVE_ATOMIC64 1
+# define ETHR_NATIVE_ATOMIC64_IMPL "ethread"
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_ATMC_T__ ethr_native_atomic64_t
+# define ETHR_AINT_T__ ethr_sint64_t
+# define ETHR_AINT_SUFFIX__ "q"
+# else
+# error "Unsupported integer size"
+# endif
/* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations.
*/
@@ -81,87 +70,28 @@ typedef struct {
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
{
return (ETHR_AINT_T__ *) &var->counter;
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
- var->counter = i;
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
- var->counter = i;
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
-{
- return var->counter;
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- __asm__ __volatile__(
- "lock; add" ETHR_AINT_SUFFIX__ " %1, %0"
- : "=m"(var->counter)
- : "ir"(incr), "m"(var->counter));
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
- __asm__ __volatile__(
- "lock; inc" ETHR_AINT_SUFFIX__ " %0"
- : "=m"(var->counter)
- : "m"(var->counter));
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
- __asm__ __volatile__(
- "lock; dec" ETHR_AINT_SUFFIX__ " %0"
- : "=m"(var->counter)
- : "m"(var->counter));
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- ETHR_AINT_T__ tmp;
-
- tmp = incr;
- __asm__ __volatile__(
- "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */
- : "=r"(tmp)
- : "m"(var->counter), "0"(tmp));
- /* now tmp is the atomic's previous value */
- return tmp + incr;
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+ ETHR_AINT_T__ new,
+ ETHR_AINT_T__ old)
{
__asm__ __volatile__(
"lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3"
@@ -171,110 +101,148 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
return old;
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- ETHR_AINT_T__ tmp, old;
-
- tmp = var->counter;
- do {
- old = tmp;
- tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp);
- } while (__builtin_expect(tmp != old, 0));
- /* now tmp is the atomic's previous value */
- return tmp;
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- ETHR_AINT_T__ tmp, old;
-
- tmp = var->counter;
- do {
- old = tmp;
- tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp);
- } while (__builtin_expect(tmp != old, 0));
- /* now tmp is the atomic's previous value */
- return tmp;
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
+ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
{
ETHR_AINT_T__ tmp = val;
__asm__ __volatile__(
"xchg" ETHR_AINT_SUFFIX__ " %0, %1"
: "=r"(tmp)
- : "m"(var->counter), "0"(tmp));
+ : "m"(var->counter), "0"(tmp)
+ : "memory");
/* now tmp is the atomic's previous value */
return tmp;
}
-/*
- * Atomic ops with at least specified barriers.
- */
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
{
- ETHR_AINT_T__ val;
-#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
- val = var->counter;
+ var->counter = i;
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
#else
- val = ETHR_NATMC_FUNC__(add_return)(var, 0);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
#endif
- __asm__ __volatile__("" : : : "memory");
- return val;
-}
static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
{
- __asm__ __volatile__("" : : : "memory");
-#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
- var->counter = i;
+#if defined(_M_IX86)
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i);
+ else
+#endif /* _M_IX86 */
+ {
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ var->counter = i;
+ }
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
#else
- (void) ETHR_NATMC_FUNC__(xchg)(var, i);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+ (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i);
}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
- ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var);
- __asm__ __volatile__("" : : : "memory");
- return res;
+ return var->counter;
}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB 1
+#endif
+
static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(add_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
- __asm__ __volatile__("" : : : "memory");
- ETHR_NATMC_FUNC__(dec)(var);
-}
+ __asm__ __volatile__(
+ "lock; add" ETHR_AINT_SUFFIX__ " %1, %0"
+ : "=m"(var->counter)
+ : "ir"(incr), "m"(var->counter)
+ : "memory");
+}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(inc_mb)(ETHR_ATMC_T__ *var)
{
- __asm__ __volatile__("" : : : "memory");
- return ETHR_NATMC_FUNC__(dec_return)(var);
+ __asm__ __volatile__(
+ "lock; inc" ETHR_AINT_SUFFIX__ " %0"
+ : "=m"(var->counter)
+ : "m"(var->counter)
+ : "memory");
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(dec_mb)(ETHR_ATMC_T__ *var)
{
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+ __asm__ __volatile__(
+ "lock; dec" ETHR_AINT_SUFFIX__ " %0"
+ : "=m"(var->counter)
+ : "m"(var->counter)
+ : "memory");
}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+ ETHR_AINT_T__ tmp;
+
+ tmp = incr;
+ __asm__ __volatile__(
+ "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */
+ : "=r"(tmp)
+ : "m"(var->counter), "0"(tmp)
+ : "memory");
+ /* now tmp is the atomic's previous value */
+ return tmp + incr;
}
#endif /* ETHR_TRY_INLINE_FUNCS */
diff --git a/erts/include/internal/i386/ethr_dw_atomic.h b/erts/include/internal/i386/ethr_dw_atomic.h
new file mode 100644
index 0000000000..9fb89bbe43
--- /dev/null
+++ b/erts/include/internal/i386/ethr_dw_atomic.h
@@ -0,0 +1,278 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics for x86/x86_64
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_X86_DW_ATOMIC_H__
+#define ETHR_X86_DW_ATOMIC_H__
+
+#ifdef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT
+
+#define ETHR_HAVE_NATIVE_DW_ATOMIC
+#define ETHR_NATIVE_DW_ATOMIC_IMPL "ethread"
+
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+#define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+ ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+
+#if ETHR_SIZEOF_PTR == 4
+typedef volatile ethr_sint64_t * ethr_native_dw_ptr_t;
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+# define ETHR_DW_CMPXCHG_SFX__ "8b"
+# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+#else
+#ifdef ETHR_HAVE_INT128_T
+# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t
+typedef volatile ethr_sint128_t * ethr_native_dw_ptr_t;
+#else
+typedef struct {
+ ethr_sint64_t sint64[2];
+} ethr_native_sint128_t__;
+typedef volatile ethr_native_sint128_t__ * ethr_native_dw_ptr_t;
+#endif
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+# define ETHR_DW_CMPXCHG_SFX__ "16b"
+#endif
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+ (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+#ifdef ETHR_NATIVE_SU_DW_SINT_T
+ volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+#endif
+ volatile ethr_sint_t sint[3];
+ volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+
+#if (defined(ETHR_TRY_INLINE_FUNCS) \
+ || defined(ETHR_ATOMIC_IMPL__) \
+ || defined(ETHR_X86_SSE2_ASM_C__)) \
+ && ETHR_SIZEOF_PTR == 4 \
+ && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+ethr_sint64_t
+ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var);
+void
+ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+ ethr_sint64_t val);
+#endif
+
+#if (defined(ETHR_TRY_INLINE_FUNCS) \
+ || defined(ETHR_ATOMIC_IMPL__) \
+ || defined(ETHR_X86_SSE2_ASM_C__))
+# ifdef ETHR_DEBUG
+# define ETHR_DW_DBG_ALIGNED__(PTR) \
+ ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+# else
+# define ETHR_DW_DBG_ALIGNED__(PTR)
+# endif
+#endif
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+ return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+}
+
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+/*
+ * When position independent code is used in 32-bit mode, the EBX register
+ * is used for storage of global offset table address, and we may not
+ * use it as input or output in an asm. We need to save and restore the
+ * EBX register explicitly (for some reason gcc doesn't provide this
+ * service to us).
+ */
+# define ETHR_NO_CLOBBER_EBX__ 1
+#else
+# define ETHR_NO_CLOBBER_EBX__ 0
+#endif
+
+#if ETHR_NO_CLOBBER_EBX__ && !defined(ETHR_CMPXCHG8B_REGISTER_SHORTAGE)
+/* When no optimization is on, we'll run into a register shortage */
+# if defined(ETHR_DEBUG) || defined(DEBUG) || defined(VALGRIND) \
+ || defined(GCOV) || defined(PURIFY) || defined(PURECOV)
+# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 1
+# else
+# define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 0
+# endif
+#endif
+
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE int
+ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+ ethr_sint_t *new,
+ ethr_sint_t *xchg)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ char xchgd;
+
+ ETHR_DW_DBG_ALIGNED__(p);
+
+ __asm__ __volatile__(
+#if ETHR_NO_CLOBBER_EBX__
+ "pushl %%ebx\n\t"
+# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE
+ "movl (%7), %%ebx\n\t"
+ "movl 4(%7), %%ecx\n\t"
+# else
+ "movl %8, %%ebx\n\t"
+# endif
+#endif
+ "lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t"
+ "setz %3\n\t"
+#if ETHR_NO_CLOBBER_EBX__
+ "popl %%ebx\n\t"
+#endif
+ : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+ : "m"(*p), "1"(xchg[1]), "2"(xchg[0]),
+#if ETHR_NO_CLOBBER_EBX__
+# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE
+ "3"(new)
+# else
+ "3"(new[1]),
+ "r"(new[0])
+# endif
+#else
+ "3"(new[1]),
+ "b"(new[0])
+#endif
+ : "cc", "memory");
+
+ return (int) xchgd;
+}
+
+#undef ETHR_NO_CLOBBER_EBX__
+
+#if ETHR_SIZEOF_PTR == 4 && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+
+typedef union {
+ ethr_sint64_t sint64;
+ ethr_sint_t sint[2];
+} ethr_dw_atomic_no_sse2_convert_t;
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+
+static ETHR_INLINE ethr_sint64_t
+ethr_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var)
+{
+ if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__)
+ return ethr_sse2_native_su_dw_atomic_read(var);
+ else {
+ ethr_sint_t new[2];
+ ethr_dw_atomic_no_sse2_convert_t xchg;
+ new[0] = new[1] = xchg.sint[0] = xchg.sint[1] = 0x83838383;
+ (void) ethr_native_dw_atomic_cmpxchg_mb(var, new, xchg.sint);
+ return xchg.sint64;
+ }
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+
+static ETHR_INLINE void
+ethr_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+ ethr_sint64_t val)
+{
+ if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__)
+ ethr_sse2_native_su_dw_atomic_set(var, val);
+ else {
+ ethr_sint_t xchg[2] = {0, 0};
+ ethr_dw_atomic_no_sse2_convert_t new;
+ new.sint64 = val;
+ while (!ethr_native_dw_atomic_cmpxchg_mb(var, new.sint, xchg));
+ }
+}
+
+#endif /* ETHR_SIZEOF_PTR == 4 */
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#if defined(ETHR_X86_SSE2_ASM_C__) \
+ && ETHR_SIZEOF_PTR == 4 \
+ && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+
+/*
+ * 8-byte aligned loads and stores of 64-bit values are atomic from
+ * pentium and forward. An ordinary volatile load or store in 32-bit
+ * mode generates two 32-bit operations (at least with gcc-4.1.2 using
+ * -msse2). In order to guarantee one 64-bit load/store operation
+ * from/to memory we load/store via an xmm register using movq.
+ *
+ * Load/store can be achieved using cmpxchg8b, however, using movq is
+ * much faster. Unfortunately we cannot do the same thing in 64-bit
+ * mode; instead, we have to do loads and stores via cmpxchg16b.
+ *
+ * We do not inline these, but instead compile these into a separate
+ * object file using -msse2. This since we don't want to use -msse2 for
+ * the whole system. If we detect sse2 support (pentium4 and forward)
+ * at runtime, we use them; otherwise, we fall back to using cmpxchg8b
+ * for loads and stores. This way the binary can be moved between
+ * processors with and without sse2 support.
+ */
+
+ethr_sint64_t
+ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ ethr_sint64_t val;
+ ETHR_DW_DBG_ALIGNED__(p);
+ __asm__ __volatile__("movq %1, %0\n\t" : "=x"(val) : "m"(*p) : "memory");
+ return val;
+}
+
+void
+ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+ ethr_sint64_t val)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ ETHR_DW_DBG_ALIGNED__(p);
+ __asm__ __volatile__("movq %1, %0\n\t" : "=m"(*p) : "x"(val) : "memory");
+}
+
+#endif /* ETHR_X86_SSE2_ASM_C__ */
+
+#endif /* ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */
+
+#endif /* ETHR_X86_DW_ATOMIC_H__ */
+
diff --git a/erts/include/internal/i386/ethr_membar.h b/erts/include/internal/i386/ethr_membar.h
new file mode 100644
index 0000000000..92d9de7f3f
--- /dev/null
+++ b/erts/include/internal/i386/ethr_membar.h
@@ -0,0 +1,114 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for x86/x86-64
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_X86_MEMBAR_H__
+#define ETHR_X86_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+#define ETHR_NO_SSE2_MEMORY_BARRIER__ \
+do { \
+ volatile ethr_sint32_t x__ = 0; \
+ __asm__ __volatile__ ("lock; orl $0x0, %0\n\t" \
+ : "=m"(x__) \
+ : "m"(x__) \
+ : "memory"); \
+} while (0)
+
+static __inline__ void
+ethr_cfence__(void)
+{
+ __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_mfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MEMORY_BARRIER__;
+ else
+#endif
+ __asm__ __volatile__ ("mfence\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_sfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MEMORY_BARRIER__;
+ else
+#endif
+ __asm__ __volatile__ ("sfence\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_lfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MEMORY_BARRIER__;
+ else
+#endif
+ __asm__ __volatile__ ("lfence\n\t" : : : "memory");
+}
+
+#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \
+ ethr_sfence__(), \
+ ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \
+ ethr_lfence__(), \
+ ethr_mfence__()))
+
+#ifdef ETHR_X86_OUT_OF_ORDER
+
+#define ETHR_MEMBAR(B) \
+ ETHR_X86_OUT_OF_ORDER_MEMBAR((B))
+
+#else /* !ETHR_X86_OUT_OF_ORDER (the default) */
+
+/*
+ * We assume that only stores before loads may be reordered. That is,
+ * we assume that *no* instructions like these are used:
+ * - CLFLUSH,
+ * - streaming stores executed with non-temporal move,
+ * - string operations, or
+ * - other instructions which aren't LoadLoad, LoadStore, and StoreStore
+ * ordered by themselves
+ * If such instructions are used, either insert memory barriers
+ * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or
+ * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32
+ * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2.
+ */
+
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__())
+
+#endif /* !ETHR_X86_OUT_OF_ORDER */
+
+#endif /* ETHR_X86_MEMBAR_H__ */
diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h
index b5a17caefb..80e4dc7b99 100644
--- a/erts/include/internal/i386/ethread.h
+++ b/erts/include/internal/i386/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,17 +24,15 @@
#ifndef ETHREAD_I386_ETHREAD_H
#define ETHREAD_I386_ETHREAD_H
+#include "ethr_membar.h"
#define ETHR_ATOMIC_WANT_32BIT_IMPL__
#include "atomic.h"
#if ETHR_SIZEOF_PTR == 8
# define ETHR_ATOMIC_WANT_64BIT_IMPL__
# include "atomic.h"
#endif
+#include "ethr_dw_atomic.h"
#include "spinlock.h"
#include "rwlock.h"
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_SPINLOCKS 1
-#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
-
#endif /* ETHREAD_I386_ETHREAD_H */
diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h
index be47f459ce..1a8cd7da0c 100644
--- a/erts/include/internal/i386/rwlock.h
+++ b/erts/include/internal/i386/rwlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -26,6 +26,9 @@
#ifndef ETHREAD_I386_RWLOCK_H
#define ETHREAD_I386_RWLOCK_H
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
/* XXX: describe the algorithm */
typedef struct {
volatile int lock;
diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h
index 0325324895..a84fba91b1 100644
--- a/erts/include/internal/i386/spinlock.h
+++ b/erts/include/internal/i386/spinlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,9 @@
#ifndef ETHREAD_I386_SPINLOCK_H
#define ETHREAD_I386_SPINLOCK_H
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
/* A spinlock is the low byte of an aligned 32-bit integer.
* A non-zero value means that the lock is locked.
*/
@@ -46,16 +49,20 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
* On i386 this needs to be a locked operation
* to avoid Pentium Pro errata 66 and 92.
*/
-#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
- __asm__ __volatile__("" : : : "memory");
- *(unsigned char*)&lock->lock = 0;
-#else
- char tmp = 0;
- __asm__ __volatile__(
- "xchgb %b0, %1"
- : "=q"(tmp), "=m"(lock->lock)
- : "0"(tmp) : "memory");
+#if !defined(__x86_64__)
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) {
+ char tmp = 0;
+ __asm__ __volatile__(
+ "xchgb %b0, %1"
+ : "=q"(tmp), "=m"(lock->lock)
+ : "0"(tmp) : "memory");
+ }
+ else
#endif
+ {
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ *(unsigned char*)&lock->lock = 0;
+ }
}
static ETHR_INLINE int
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index 93bc06036f..fb1288c330 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -25,16 +25,6 @@
#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
#define ETHR_LIBATOMIC_OPS_ATOMIC_H__
-#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS)
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-
-#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \
- || defined(__x86_64__)
-#define AO_USE_PENTIUM4_INSTRS
-#endif
-
-#include "atomic_ops.h"
-
/*
* libatomic_ops can be downloaded from:
* http://www.hpl.hp.com/research/linux/atomic_ops/
@@ -46,21 +36,18 @@
* - AO_store()
* - AO_compare_and_swap()
*
- * The `AO_t' type also have to be at least as large as the `void *' type.
*/
-#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR
-#error The AO_t type is too small
-#endif
-
#if ETHR_SIZEOF_AO_T == 4
#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic32_t
#define ETHR_AINT_T__ ethr_sint32_t
#define ETHR_AINT_SUFFIX__ "l"
#elif ETHR_SIZEOF_AO_T == 8
#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic64_t
#define ETHR_AINT_T__ ethr_sint64_t
@@ -69,61 +56,29 @@
#error "Unsupported integer size"
#endif
-#if ETHR_SIZEOF_AO_T == 8
-typedef union {
- volatile AO_t counter;
- ethr_sint32_t sint32[2];
-} ETHR_ATMC_T__;
-#else
typedef struct {
volatile AO_t counter;
} ETHR_ATMC_T__;
-#endif
-#define ETHR_MEMORY_BARRIER AO_nop_full()
-#ifdef AO_HAVE_nop_write
-# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write()
-#else
-# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_HAVE_nop_read
-# define ETHR_READ_MEMORY_BARRIER AO_nop_read()
-#else
-# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_NO_DD_ORDERING
-# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
#else
-# define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier()
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
#endif
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-
static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
{
return (ETHR_AINT_T__ *) &var->counter;
}
-#if ETHR_SIZEOF_AO_T == 8
-/*
- * We also need to provide an ethr_native_atomic32_addr(), since
- * this 64-bit implementation will be used implementing 32-bit
- * native atomics.
- */
-
-static ETHR_INLINE ethr_sint32_t *
-ethr_native_atomic32_addr(ETHR_ATMC_T__ *var)
-{
- ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter));
-#ifdef ETHR_BIGENDIAN
- return &var->sint32[1];
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
#else
- return &var->sint32[0];
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
#endif
-}
-
-#endif /* ETHR_SIZEOF_AO_T == 8 */
static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
@@ -131,197 +86,198 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
AO_store(&var->counter, (AO_t) value);
}
+#ifdef AO_HAVE_store_release
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
- ETHR_NATMC_FUNC__(set)(var, value);
+ AO_store_release(&var->counter, (AO_t) value);
}
+#endif
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
return (ETHR_AINT_T__) AO_load(&var->counter);
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
-#ifdef AO_HAVE_fetch_and_add_full
- return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
+#ifdef AO_HAVE_load_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
#else
- while (1) {
- AO_t exp = AO_load(&var->counter);
- AO_t new = exp + (AO_t) incr;
- if (AO_compare_and_swap_full(&var->counter, exp, new))
- return (ETHR_AINT_T__) new;
- }
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
{
- (void) ETHR_NATMC_FUNC__(add_return)(var, incr);
+ return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_add1_full
- return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
-#else
- return ETHR_NATMC_FUNC__(add_return)(var, 1);
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
- (void) ETHR_NATMC_FUNC__(inc_return)(var);
-}
+#ifdef AO_HAVE_fetch_and_add
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_sub1_full
- return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
#else
- return ETHR_NATMC_FUNC__(add_return)(var, -1);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
#endif
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
- (void) ETHR_NATMC_FUNC__(dec_return)(var);
-}
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
- while (1) {
- AO_t exp = AO_load(&var->counter);
- AO_t new = exp & ((AO_t) mask);
- if (AO_compare_and_swap_full(&var->counter, exp, new))
- return (ETHR_AINT_T__) exp;
- }
+ return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- while (1) {
- AO_t exp = AO_load(&var->counter);
- AO_t new = exp | ((AO_t) mask);
- if (AO_compare_and_swap_full(&var->counter, exp, new))
- return (ETHR_AINT_T__) exp;
- }
-}
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ exp)
-{
- ETHR_AINT_T__ act;
- do {
- if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
- return exp;
- act = (ETHR_AINT_T__) AO_load(&var->counter);
- } while (act == exp);
- return act;
-}
+#ifdef AO_HAVE_fetch_and_add1
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
{
- while (1) {
- AO_t exp = AO_load(&var->counter);
- if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new))
- return (ETHR_AINT_T__) exp;
- }
+ return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
}
-/*
- * Atomic ops with at least specified barriers.
- */
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_load_acquire
- return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#ifdef AO_HAVE_fetch_and_add1_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
#else
- ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
- ETHR_MEMORY_BARRIER;
- return res;
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
{
-#ifdef AO_HAVE_fetch_and_add1_acquire
return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
#else
- ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1);
- return res;
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
{
-#ifdef AO_HAVE_store_release
- AO_store_release(&var->counter, (AO_t) value);
+ return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1_release
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
#else
- ETHR_MEMORY_BARRIER;
- ETHR_NATMC_FUNC__(set)(var, value);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
{
-#ifdef AO_HAVE_fetch_and_sub1_release
return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
#else
- return ETHR_NATMC_FUNC__(dec_return)(var);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+ ETHR_AINT_T__ new,
+ ETHR_AINT_T__ exp)
{
- (void) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+ ETHR_AINT_T__ act;
+ do {
+ if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
+ return exp;
+ act = (ETHR_AINT_T__) AO_load(&var->counter);
+ } while (act == exp);
+ return act;
}
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
ETHR_AINT_T__ new,
ETHR_AINT_T__ exp)
{
-#ifdef AO_HAVE_compare_and_swap_acquire
ETHR_AINT_T__ act;
do {
if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
return exp;
+#ifdef AO_HAVE_load_acquire
+ act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
act = (ETHR_AINT_T__) AO_load(&var->counter);
+#endif
} while (act == exp);
+#ifndef AO_HAVE_load_acquire
AO_nop_full();
+#endif
return act;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_release
+
+#if ETHR_SIZEOF_AO_T == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
#else
- ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
- return act;
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
ETHR_AINT_T__ new,
ETHR_AINT_T__ exp)
{
-#ifdef AO_HAVE_compare_and_swap_release
ETHR_AINT_T__ act;
do {
if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
@@ -329,11 +285,9 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
act = (ETHR_AINT_T__) AO_load(&var->counter);
} while (act == exp);
return act;
-#else
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-#endif
}
+#endif
#endif /* ETHR_TRY_INLINE_FUNCS */
@@ -341,6 +295,4 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
#undef ETHR_ATMC_T__
#undef ETHR_AINT_T__
-#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */
-
#endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */
diff --git a/erts/include/internal/libatomic_ops/ethr_membar.h b/erts/include/internal/libatomic_ops/ethr_membar.h
new file mode 100644
index 0000000000..b8530a0094
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethr_membar.h
@@ -0,0 +1,75 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers when using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_LIBATOMIC_OPS_MEMBAR_H__
+#define ETHR_LIBATOMIC_OPS_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+#ifndef AO_HAVE_nop_full
+# error "No AO_nop_full()"
+#endif
+
+static __inline__ void
+ethr_mb__(void)
+{
+ AO_nop_full();
+}
+
+static __inline__ void
+ethr_rb__(void)
+{
+#ifdef AO_HAVE_nop_read
+ AO_nop_read();
+#else
+ AO_nop_full();
+#endif
+}
+
+static __inline__ void
+ethr_wb__(void)
+{
+#ifdef AO_HAVE_nop_write
+ AO_nop_write();
+#else
+ AO_nop_full();
+#endif
+}
+
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \
+ ethr_wb__(), \
+ ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \
+ ethr_rb__(), \
+ ethr_mb__()))
+
+#define ETHR_COMPILER_BARRIER AO_compiler_barrier()
+#ifdef AO_NO_DD_ORDERING
+# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#endif
+
+#endif /* ETHR_LIBATOMIC_OPS_MEMBAR_H__ */
diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h
index ee73ba73bc..e1fdd588bb 100644
--- a/erts/include/internal/libatomic_ops/ethread.h
+++ b/erts/include/internal/libatomic_ops/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,18 @@
#ifndef ETHREAD_LIBATOMIC_OPS_H__
#define ETHREAD_LIBATOMIC_OPS_H__
+#if (defined(ETHR_HAVE_LIBATOMIC_OPS) \
+ && ((ETHR_SIZEOF_AO_T == 4 && !defined(ETHR_HAVE_NATIVE_ATOMIC32)) \
+ || (ETHR_SIZEOF_AO_T == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64))))
+
+#if defined(__x86_64__)
+#define AO_USE_PENTIUM4_INSTRS
+#endif
+
+#include "atomic_ops.h"
+#include "ethr_membar.h"
#include "ethr_atomic.h"
#endif
+
+#endif
diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index 522f433649..6001620677 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -29,27 +29,31 @@
#define ETHREAD_PPC_ATOMIC_H
#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
typedef struct {
volatile ethr_sint32_t counter;
} ethr_native_atomic32_t;
-#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
-
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+
static ETHR_INLINE ethr_sint32_t *
ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
{
return (ethr_sint32_t *) &var->counter;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+
static ETHR_INLINE void
-ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
+ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
{
var->counter = i;
}
-#define ethr_native_atomic32_set(v, i) ethr_native_atomic32_init((v), (i))
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_read(ethr_native_atomic32_t *var)
@@ -57,57 +61,68 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var)
return var->counter;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
{
ethr_sint32_t tmp;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%1\n\t"
"add %0,%2,%0\n\t"
"stwcx. %0,0,%1\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(tmp)
: "r"(&var->counter), "r"(incr)
: "cc", "memory");
return tmp;
}
-static ETHR_INLINE void
-ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr)
{
- /* XXX: could use weaker version here w/o eieio+isync */
- (void)ethr_native_atomic32_add_return(var, incr);
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_add_return(var, incr);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
{
ethr_sint32_t tmp;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%1\n\t"
"addic %0,%0,1\n\t" /* due to addi's (rA|0) behaviour */
"stwcx. %0,0,%1\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(tmp)
: "r"(&var->counter)
: "cc", "memory");
return tmp;
}
-static ETHR_INLINE void
-ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
{
- /* XXX: could use weaker version here w/o eieio+isync */
- (void)ethr_native_atomic32_inc_return(var);
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_inc_return(var);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
@@ -115,82 +130,120 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
ethr_sint32_t tmp;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%1\n\t"
"addic %0,%0,-1\n\t"
"stwcx. %0,0,%1\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(tmp)
: "r"(&var->counter)
: "cc", "memory");
return tmp;
}
-static ETHR_INLINE void
-ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var)
{
- /* XXX: could use weaker version here w/o eieio+isync */
- (void)ethr_native_atomic32_dec_return(var);
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_dec_return(var);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
{
ethr_sint32_t old, new;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%2\n\t"
"and %1,%0,%3\n\t"
"stwcx. %1,0,%2\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(old), "=&r"(new)
: "r"(&var->counter), "r"(mask)
: "cc", "memory");
return old;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_and_retold(var, mask);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
{
ethr_sint32_t old, new;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%2\n\t"
"or %1,%0,%3\n\t"
"stwcx. %1,0,%2\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(old), "=&r"(new)
: "r"(&var->counter), "r"(mask)
: "cc", "memory");
return old;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_or_retold(var, mask);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
{
ethr_sint32_t tmp;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%1\n\t"
"stwcx. %2,0,%1\n\t"
"bne- 1b\n\t"
- "isync"
: "=&r"(tmp)
: "r"(&var->counter), "r"(val)
: "cc", "memory");
return tmp;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ res = ethr_native_atomic32_xchg(var, val);
+ __asm__ __volatile("isync\n\t" : : : "memory");
+ return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
ethr_sint32_t new,
@@ -199,14 +252,12 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
ethr_sint32_t old;
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%2\n\t"
"cmpw 0,%0,%3\n\t"
"bne 2f\n\t"
"stwcx. %1,0,%2\n\t"
"bne- 1b\n\t"
- "isync\n"
"2:"
: "=&r"(old)
: "r"(new), "r"(&var->counter), "r"(expected)
@@ -215,25 +266,30 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
return old;
}
-/*
- * Atomic ops with at least specified barriers.
- */
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
-static ETHR_INLINE long
-ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+ ethr_sint32_t new,
+ ethr_sint32_t expected)
{
- long res = ethr_native_atomic32_read(var);
- ETHR_MEMORY_BARRIER;
- return res;
-}
+ ethr_sint32_t old;
-#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg
-#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return
-#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return
-#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return
+ __asm__ __volatile__(
+ "1:\t"
+ "lwarx %0,0,%2\n\t"
+ "cmpw 0,%0,%3\n\t"
+ "bne 2f\n\t"
+ "stwcx. %1,0,%2\n\t"
+ "bne- 1b\n\t"
+ "isync\n"
+ "2:"
+ : "=&r"(old)
+ : "r"(new), "r"(&var->counter), "r"(expected)
+ : "cc", "memory");
-#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg
-#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg
+ return old;
+}
#endif /* ETHR_TRY_INLINE_FUNCS */
diff --git a/erts/include/internal/ppc32/ethr_membar.h b/erts/include/internal/ppc32/ethr_membar.h
new file mode 100644
index 0000000000..ff5cc86bfb
--- /dev/null
+++ b/erts/include/internal/ppc32/ethr_membar.h
@@ -0,0 +1,63 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for PowerPC
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_PPC_MEMBAR_H__
+#define ETHR_PPC_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+static __inline__ void
+ethr_lwsync__(void)
+{
+#ifdef ETHR_PPC_HAVE_NO_LWSYNC
+ __asm__ __volatile__ ("sync\n\t" : : : "memory");
+#else
+#ifndef ETHR_PPC_HAVE_LWSYNC
+ if (ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__)
+ __asm__ __volatile__ ("sync\n\t" : : : "memory");
+ else
+#endif
+ __asm__ __volatile__ ("lwsync\n\t" : : : "memory");
+#endif
+}
+
+static __inline__ void
+ethr_sync__(void)
+{
+ __asm__ __volatile__ ("sync\n\t" : : : "memory");
+}
+
+/*
+ * According to the "memory barrier intstructions" section of
+ * http://www.ibm.com/developerworks/systems/articles/powerpc.html
+ * we want to use sync when a StoreLoad is needed and lwsync for
+ * everything else.
+ */
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_sync__(), ethr_lwsync__())
+
+#endif
diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h
index 3b619e9d01..e41c83c5da 100644
--- a/erts/include/internal/ppc32/ethread.h
+++ b/erts/include/internal/ppc32/ethread.h
@@ -24,12 +24,9 @@
#ifndef ETHREAD_PPC32_ETHREAD_H
#define ETHREAD_PPC32_ETHREAD_H
+#include "ethr_membar.h"
#include "atomic.h"
#include "spinlock.h"
#include "rwlock.h"
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_SPINLOCKS 1
-#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
-
#endif /* ETHREAD_PPC32_ETHREAD_H */
diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h
index 19ec26ab68..311f000b69 100644
--- a/erts/include/internal/ppc32/rwlock.h
+++ b/erts/include/internal/ppc32/rwlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -23,12 +23,14 @@
*
* Based on the examples in Appendix E of Motorola's
* "Programming Environments Manual For 32-Bit Implementations
- * of the PowerPC Architecture". Uses eieio instead of sync
- * in the unlock sequence, as suggested in the manual.
+ * of the PowerPC Architecture".
*/
#ifndef ETHREAD_PPC_RWLOCK_H
#define ETHREAD_PPC_RWLOCK_H
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
/* Unlocked if zero, read-locked if negative, write-locked if +1. */
typedef struct {
volatile int lock;
@@ -47,9 +49,10 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock)
{
int tmp;
- /* this is eieio + ethr_native_atomic_inc() - isync */
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+
+ /* this is ethr_native_atomic_inc() - isync */
__asm__ __volatile__(
- "eieio\n\t"
"1:\t"
"lwarx %0,0,%1\n\t"
"addic %0,%0,1\n\t"
@@ -105,7 +108,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
static ETHR_INLINE void
ethr_native_write_unlock(ethr_native_rwlock_t *lock)
{
- __asm__ __volatile__("eieio" : : : "memory");
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
lock->lock = 0;
}
diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h
index c8460a3e8a..4c95ec9efb 100644
--- a/erts/include/internal/ppc32/spinlock.h
+++ b/erts/include/internal/ppc32/spinlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -23,12 +23,14 @@
*
* Based on the examples in Appendix E of Motorola's
* "Programming Environments Manual For 32-Bit Implementations
- * of the PowerPC Architecture". Uses eieio instead of sync
- * in the unlock sequence, as suggested in the manual.
+ * of the PowerPC Architecture".
*/
#ifndef ETHREAD_PPC_SPINLOCK_H
#define ETHREAD_PPC_SPINLOCK_H
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
/* Unlocked if zero, locked if non-zero. */
typedef struct {
volatile unsigned int lock;
@@ -45,7 +47,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
static ETHR_INLINE void
ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
- __asm__ __volatile__("eieio" : : : "memory");
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
lock->lock = 0;
}
diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h
index 4c29b28536..d0a77990cc 100644
--- a/erts/include/internal/pthread/ethr_event.h
+++ b/erts/include/internal/pthread/ethr_event.h
@@ -21,7 +21,7 @@
* Author: Rickard Green
*/
-#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS)
+#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
/* --- Linux futex implementation of ethread events ------------------------- */
#define ETHR_LINUX_FUTEX_IMPL__
@@ -62,8 +62,7 @@ static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
{
ethr_sint32_t val;
- ETHR_MEMORY_BARRIER;
- val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__);
+ val = ethr_atomic32_xchg_mb(&e->futex, ETHR_EVENT_ON__);
if (val == ETHR_EVENT_OFF_WAITER__) {
int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1);
if (res != 0)
@@ -99,8 +98,7 @@ static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
{
ethr_sint32_t val;
- ETHR_MEMORY_BARRIER;
- val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__);
+ val = ethr_atomic32_xchg_mb(&e->state, ETHR_EVENT_ON__);
if (val == ETHR_EVENT_OFF_WAITER__) {
int res = pthread_mutex_lock(&e->mtx);
if (res != 0)
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index c297522ab1..fe1daaa9cf 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -35,23 +35,16 @@
#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
-#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__
-#define ETHR_SPARC_V9_ATOMIC_COMMON__
-
-#define ETHR_MEMORY_BARRIER \
- __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \
- : : : "memory")
-
-#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */
-
#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic32_t
#define ETHR_AINT_T__ ethr_sint32_t
#define ETHR_CAS__ "cas"
#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "ethread"
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic64_t
#define ETHR_AINT_T__ ethr_sint64_t
@@ -66,17 +59,23 @@ typedef struct {
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
{
return (ETHR_AINT_T__ *) &var->counter;
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
- var->counter = i;
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
@@ -84,182 +83,49 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
var->counter = i;
}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
return var->counter;
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- ETHR_AINT_T__ old, tmp;
-
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
- do {
- old = var->counter;
- tmp = old+incr;
- __asm__ __volatile__(
- ETHR_CAS__ " [%2], %1, %0"
- : "=&r"(tmp)
- : "r"(old), "r"(&var->counter), "0"(tmp)
- : "memory");
- } while (__builtin_expect(old != tmp, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
- return old+incr;
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- (void)ETHR_NATMC_FUNC__(add_return)(var, incr);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(add_return)(var, 1);
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
- (void)ETHR_NATMC_FUNC__(add_return)(var, 1);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(add_return)(var, -1);
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
- (void)ETHR_NATMC_FUNC__(add_return)(var, -1);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- ETHR_AINT_T__ old, tmp;
-
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
- do {
- old = var->counter;
- tmp = old & mask;
- __asm__ __volatile__(
- ETHR_CAS__ " [%2], %1, %0"
- : "=&r"(tmp)
- : "r"(old), "r"(&var->counter), "0"(tmp)
- : "memory");
- } while (__builtin_expect(old != tmp, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
- return old;
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
- ETHR_AINT_T__ old, tmp;
-
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
- do {
- old = var->counter;
- tmp = old | mask;
- __asm__ __volatile__(
- ETHR_CAS__ " [%2], %1, %0"
- : "=&r"(tmp)
- : "r"(old), "r"(&var->counter), "0"(tmp)
- : "memory");
- } while (__builtin_expect(old != tmp, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
- return old;
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
-{
- ETHR_AINT_T__ old, new;
-
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory");
- do {
- old = var->counter;
- new = val;
- __asm__ __volatile__(
- ETHR_CAS__ " [%2], %1, %0"
- : "=&r"(new)
- : "r"(old), "r"(&var->counter), "0"(new)
- : "memory");
- } while (__builtin_expect(old != new, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
- return old;
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
{
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
__asm__ __volatile__(
ETHR_CAS__ " [%2], %1, %0"
: "=&r"(new)
: "r"(old), "r"(&var->counter), "0"(new)
: "memory");
- __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
return new;
}
-/*
- * Atomic ops with at least specified barriers.
- */
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
- ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
- __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
- return res;
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
- __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
- ETHR_NATMC_FUNC__(set)(var, i);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
-{
- ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var);
- return res;
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
-{
- ETHR_NATMC_FUNC__(dec)(var);
-}
-
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
-{
- return ETHR_NATMC_FUNC__(dec_return)(var);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
{
ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
return res;
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
-{
- return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
-}
-
#endif /* ETHR_TRY_INLINE_FUNCS */
#undef ETHR_NATMC_FUNC__
diff --git a/erts/include/internal/sparc32/ethr_membar.h b/erts/include/internal/sparc32/ethr_membar.h
new file mode 100644
index 0000000000..6eb0c5a1d6
--- /dev/null
+++ b/erts/include/internal/sparc32/ethr_membar.h
@@ -0,0 +1,115 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for sparc-v9
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_SPARC_V9_MEMBAR_H__
+#define ETHR_SPARC_V9_MEMBAR_H__
+
+#if defined(ETHR_SPARC_TSO)
+/* --- Total Store Order ------------------------------------------------ */
+
+#define ETHR_LoadLoad 0
+#define ETHR_LoadStore 0
+#define ETHR_StoreLoad 1
+#define ETHR_StoreStore 0
+
+static __inline__ void
+ethr_cb__(void)
+{
+ __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad__(void)
+{
+ __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory");
+}
+
+
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B), ethr_StoreLoad__(), ethr_cb__())
+
+#elif defined(ETHR_SPARC_PSO)
+/* --- Partial Store Order ---------------------------------------------- */
+
+#define ETHR_LoadLoad 0
+#define ETHR_LoadStore 0
+#define ETHR_StoreLoad (1 << 0)
+#define ETHR_StoreStore (1 << 1)
+
+static __inline__ void
+ethr_cb__(void)
+{
+ __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad__(void)
+{
+ __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreStore__(void)
+{
+ __asm__ __volatile__ ("membar #StoreStore\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad_StoreStore__(void)
+{
+ __asm__ __volatile__ ("membar #StoreLoad|StoreStore\n\t" : : : "memory");
+}
+
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR( \
+ (B) == ETHR_StoreLoad, \
+ ethr_StoreLoad__(), \
+ ETHR_CHOOSE_EXPR( \
+ (B) == ETHR_StoreStore, \
+ ethr_StoreStore__(), \
+ ETHR_CHOOSE_EXPR( \
+ (B) == (ETHR_StoreLoad|ETHR_StoreStore), \
+ ethr_StoreLoad_StoreStore__(), \
+ ethr_cb__())))
+
+#elif defined(ETHR_SPARC_RMO)
+/* --- Relaxed Memory Order --------------------------------------------- */
+
+# define ETHR_LoadLoad #LoadLoad
+# define ETHR_LoadStore #LoadStore
+# define ETHR_StoreLoad #StoreLoad
+# define ETHR_StoreStore #StoreStore
+
+# define ETHR_MEMBAR_AUX__(B) \
+ __asm__ __volatile__("membar " #B "\n\t" : : : "memory")
+
+# define ETHR_MEMBAR(B) ETHR_MEMBAR_AUX__(B)
+
+#else
+
+# error "No memory order defined"
+
+#endif
+
+#endif
diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h
index aea9794390..5ad92d3da7 100644
--- a/erts/include/internal/sparc32/ethread.h
+++ b/erts/include/internal/sparc32/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,7 @@
#ifndef ETHREAD_SPARC32_ETHREAD_H
#define ETHREAD_SPARC32_ETHREAD_H
+#include "ethr_membar.h"
#define ETHR_ATOMIC_WANT_32BIT_IMPL__
#include "atomic.h"
#if ETHR_SIZEOF_PTR == 8
@@ -33,8 +34,4 @@
#include "spinlock.h"
#include "rwlock.h"
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_SPINLOCKS 1
-#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
-
#endif /* ETHREAD_SPARC32_ETHREAD_H */
diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h
index 465ec96866..8b6f2e9c57 100644
--- a/erts/include/internal/sparc32/rwlock.h
+++ b/erts/include/internal/sparc32/rwlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,9 @@
#ifndef ETHREAD_SPARC32_RWLOCK_H
#define ETHREAD_SPARC32_RWLOCK_H
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
/* Unlocked if zero, read-locked if positive, write-locked if -1. */
typedef struct {
volatile int lock;
@@ -42,7 +45,7 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock)
{
unsigned int old, new;
- __asm__ __volatile__("membar #LoadLoad|#StoreLoad");
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
do {
old = lock->lock;
new = old-1;
@@ -70,7 +73,7 @@ ethr_native_read_trylock(ethr_native_rwlock_t *lock)
: "r"(old), "r"(&lock->lock), "0"(new)
: "memory");
} while (__builtin_expect(old != new, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
return 1;
}
@@ -87,7 +90,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
if (__builtin_expect(ethr_native_read_trylock(lock) != 0, 1))
break;
do {
- __asm__ __volatile__("membar #LoadLoad");
+ ETHR_MEMBAR(ETHR_LoadLoad);
} while (ethr_native_read_is_locked(lock));
}
}
@@ -95,7 +98,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
static ETHR_INLINE void
ethr_native_write_unlock(ethr_native_rwlock_t *lock)
{
- __asm__ __volatile__("membar #LoadStore|#StoreStore");
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
lock->lock = 0;
}
@@ -115,7 +118,7 @@ ethr_native_write_trylock(ethr_native_rwlock_t *lock)
: "r"(old), "r"(&lock->lock), "0"(new)
: "memory");
} while (__builtin_expect(old != new, 0));
- __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
return 1;
}
@@ -132,7 +135,7 @@ ethr_native_write_lock(ethr_native_rwlock_t *lock)
if (__builtin_expect(ethr_native_write_trylock(lock) != 0, 1))
break;
do {
- __asm__ __volatile__("membar #LoadLoad");
+ ETHR_MEMBAR(ETHR_LoadLoad);
} while (ethr_native_write_is_locked(lock));
}
}
diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h
index 493d514210..d4e36e09cf 100644
--- a/erts/include/internal/sparc32/spinlock.h
+++ b/erts/include/internal/sparc32/spinlock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,9 @@
#ifndef ETHR_SPARC32_SPINLOCK_H
#define ETHR_SPARC32_SPINLOCK_H
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
/* Locked with ldstub, so unlocked when 0 and locked when non-zero. */
typedef struct {
volatile unsigned char lock;
@@ -40,7 +43,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
static ETHR_INLINE void
ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
- __asm__ __volatile__("membar #LoadStore|#StoreStore");
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
lock->lock = 0;
}
@@ -51,10 +54,10 @@ ethr_native_spin_trylock(ethr_native_spinlock_t *lock)
__asm__ __volatile__(
"ldstub [%1], %0\n\t"
- "membar #StoreLoad|#StoreStore"
: "=r"(prev)
: "r"(&lock->lock)
: "memory");
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
return prev == 0;
}
@@ -71,7 +74,7 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock)
if (__builtin_expect(ethr_native_spin_trylock(lock) != 0, 1))
break;
do {
- __asm__ __volatile__("membar #LoadLoad");
+ ETHR_MEMBAR(ETHR_LoadLoad);
} while (ethr_native_spin_is_locked(lock));
}
}
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 5697afda25..1f1553c346 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -25,6 +25,7 @@
#define ETHREAD_TILE_ATOMIC_H
#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "tilera"
#include <atomic.h>
@@ -34,27 +35,25 @@ typedef struct {
volatile ethr_sint32_t counter;
} ethr_native_atomic32_t;
-#define ETHR_MEMORY_BARRIER __insn_mf()
-
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+
static ETHR_INLINE ethr_sint32_t *
ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
{
return (ethr_sint32_t *) &var->counter;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT 1
+
static ETHR_INLINE void
ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
{
var->counter = i;
}
-static ETHR_INLINE void
-ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
-{
- atomic_exchange_acq(&var->counter, i);
-}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_read(ethr_native_atomic32_t *var)
@@ -62,139 +61,80 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var)
return var->counter;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+{
+ return atomic_compare_and_exchange_val_acq(&var->counter,
+ 0x81818181,
+ 0x81818181);
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD 1
+
static ETHR_INLINE void
ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
{
- ETHR_MEMORY_BARRIER;
atomic_add(&var->counter, incr);
- ETHR_MEMORY_BARRIER;
-}
-
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC 1
+
static ETHR_INLINE void
ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
{
- ETHR_MEMORY_BARRIER;
atomic_increment(&var->counter);
- ETHR_MEMORY_BARRIER;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC 1
+
static ETHR_INLINE void
ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
{
- ETHR_MEMORY_BARRIER;
atomic_decrement(&var->counter);
- ETHR_MEMORY_BARRIER;
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
{
- ethr_sint32_t res;
- ETHR_MEMORY_BARRIER;
- res = atomic_exchange_and_add(&var->counter, incr) + incr;
- ETHR_MEMORY_BARRIER;
- return res;
+ return atomic_exchange_and_add(&var->counter, incr) + incr;
}
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
-{
- return ethr_native_atomic32_add_return(var, 1);
-}
-
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
-{
- return ethr_native_atomic32_add_return(var, -1);
-}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
{
- ethr_sint32_t res;
- ETHR_MEMORY_BARRIER;
- res = atomic_and_val(&var->counter, mask);
- ETHR_MEMORY_BARRIER;
- return res;
+ return atomic_and_val(&var->counter, mask);
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
{
- ethr_sint32_t res;
- ETHR_MEMORY_BARRIER;
- res = atomic_or_val(&var->counter, mask);
- ETHR_MEMORY_BARRIER;
- return res;
+ return atomic_or_val(&var->counter, mask);
}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
{
- ETHR_MEMORY_BARRIER;
return atomic_exchange_acq(&var->counter, val);
}
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t expected)
-{
- ETHR_MEMORY_BARRIER;
- return atomic_compare_and_exchange_val_acq(&var->counter, new, expected);
-}
-
-/*
- * Atomic ops with at least specified barriers.
- */
-
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
-{
- ethr_sint32_t res = ethr_native_atomic32_read(var);
- ETHR_MEMORY_BARRIER;
- return res;
-}
-
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
-{
- return ethr_native_atomic32_inc_return(var);
-}
-
-static ETHR_INLINE void
-ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val)
-{
- ETHR_MEMORY_BARRIER;
- ethr_native_atomic32_set(var, val);
-}
-
-static ETHR_INLINE void
-ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var)
-{
- ethr_native_atomic32_dec(var);
-}
-
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var)
-{
- return ethr_native_atomic32_dec_return(var);
-}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
static ETHR_INLINE ethr_sint32_t
ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
ethr_sint32_t new,
- ethr_sint32_t exp)
+ ethr_sint32_t expected)
{
- return ethr_native_atomic32_cmpxchg(var, new, exp);
-}
-
-static ETHR_INLINE ethr_sint32_t
-ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
-{
- return ethr_native_atomic32_cmpxchg(var, new, exp);
+ return atomic_compare_and_exchange_val_acq(&var->counter, new, expected);
}
#endif /* ETHR_TRY_INLINE_FUNCS */
diff --git a/erts/include/internal/tile/ethr_membar.h b/erts/include/internal/tile/ethr_membar.h
new file mode 100644
index 0000000000..7cb4f3cf9a
--- /dev/null
+++ b/erts/include/internal/tile/ethr_membar.h
@@ -0,0 +1,35 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for TILE64/TILEPro
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_TILE_MEMBAR_H__
+#define ETHR_TILE_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+#define ETHR_MEMBAR(B) __insn_mf()
+
+#endif
diff --git a/erts/include/internal/tile/ethread.h b/erts/include/internal/tile/ethread.h
index 2de4d42bc6..7f579b50e7 100644
--- a/erts/include/internal/tile/ethread.h
+++ b/erts/include/internal/tile/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -23,8 +23,7 @@
#ifndef ETHREAD_TILE_ETHREAD_H
#define ETHREAD_TILE_ETHREAD_H
+#include "ethr_membar.h"
#include "atomic.h"
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-
#endif /* ETHREAD_TILE_ETHREAD_H */
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
index 60def01a7e..e11f1abf47 100644
--- a/erts/include/internal/win/ethr_atomic.h
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -24,364 +24,408 @@
#undef ETHR_INCLUDE_ATOMIC_IMPL__
#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
-#define ETHR_WIN_ATOMIC32_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
-#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+# define ETHR_WIN_ATOMIC32_H__
+# if (defined(ETHR_MEMBAR) \
+ && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \
+ && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE))
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+# endif
+# undef ETHR_ATOMIC_WANT_32BIT_IMPL__
#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
-#define ETHR_WIN_ATOMIC64_H__
-#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
-/* _InterlockedCompareExchange64() required... */
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+# define ETHR_WIN_ATOMIC64_H__
+# if (defined(ETHR_MEMBAR) \
+ && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \
+ || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \
+ || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL)))
+# define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+# endif
+# undef ETHR_ATOMIC_WANT_64BIT_IMPL__
#endif
-#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+
+#if !defined(_MSC_VER) || _MSC_VER < 1400
+# undef ETHR_INCLUDE_ATOMIC_IMPL__
#endif
#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
-#if defined(_MSC_VER) && _MSC_VER >= 1400
+# ifndef ETHR_WIN_ATOMIC_COMMON__
+# define ETHR_WIN_ATOMIC_COMMON__
+
+# if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
+# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+# else
+# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+# endif
+
+# endif /* ETHR_WIN_ATOMIC_COMMON__ */
+
+# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+
+# define ETHR_HAVE_NATIVE_ATOMIC32 1
+# define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked"
+
+# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT
+# define ETHR_WIN_HAVE_DEC
+# pragma intrinsic(_InterlockedDecrement)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
+# define ETHR_WIN_HAVE_DEC_REL
+# pragma intrinsic(_InterlockedDecrement_rel)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT
+# define ETHR_WIN_HAVE_INC
+# pragma intrinsic(_InterlockedIncrement)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
+# define ETHR_WIN_HAVE_INC_ACQ
+# pragma intrinsic(_InterlockedIncrement_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
+# define ETHR_WIN_HAVE_XCHG_ADD
+# pragma intrinsic(_InterlockedExchangeAdd)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
+# define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+# pragma intrinsic(_InterlockedExchangeAdd_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE
+# define ETHR_WIN_HAVE_XCHG
+# pragma intrinsic(_InterlockedExchange)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDAND
+# define ETHR_WIN_HAVE_AND
+# pragma intrinsic(_InterlockedAnd)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDOR
+# define ETHR_WIN_HAVE_OR
+# pragma intrinsic(_InterlockedOr)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
+# define ETHR_WIN_HAVE_CMPXCHG
+# pragma intrinsic(_InterlockedCompareExchange)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
+# define ETHR_WIN_HAVE_CMPXCHG_ACQ
+# pragma intrinsic(_InterlockedCompareExchange_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
+# define ETHR_WIN_HAVE_CMPXCHG_REL
+# pragma intrinsic(_InterlockedCompareExchange_rel)
+# endif
+
+# define ETHR_ILCKD__(X) _Interlocked ## X
+# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
+# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_ATMC_T__ ethr_native_atomic32_t
+# define ETHR_AINT_T__ ethr_sint32_t
+
+# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+
+# define ETHR_HAVE_NATIVE_ATOMIC64 1
+# define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked"
+
+# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
+# define ETHR_WIN_HAVE_DEC
+# pragma intrinsic(_InterlockedDecrement64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
+# define ETHR_WIN_HAVE_DEC_REL
+# pragma intrinsic(_InterlockedDecrement64_rel)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
+# define ETHR_WIN_HAVE_INC_ACQ
+# pragma intrinsic(_InterlockedIncrement64_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
+# define ETHR_WIN_HAVE_INC
+# pragma intrinsic(_InterlockedIncrement64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+# define ETHR_WIN_HAVE_XCHG_ADD
+# pragma intrinsic(_InterlockedExchangeAdd64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
+# define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+# pragma intrinsic(_InterlockedExchangeAdd64_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+# define ETHR_WIN_HAVE_XCHG
+# pragma intrinsic(_InterlockedExchange64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDAND64
+# define ETHR_WIN_HAVE_AND
+# pragma intrinsic(_InterlockedAnd64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDOR64
+# define ETHR_WIN_HAVE_OR
+# pragma intrinsic(_InterlockedOr64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+# define ETHR_WIN_HAVE_CMPXCHG
+# pragma intrinsic(_InterlockedCompareExchange64)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
+# define ETHR_WIN_HAVE_CMPXCHG_ACQ
+# pragma intrinsic(_InterlockedCompareExchange64_acq)
+# endif
+# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
+# define ETHR_WIN_HAVE_CMPXCHG_REL
+# pragma intrinsic(_InterlockedCompareExchange64_rel)
+# endif
+
+# define ETHR_ILCKD__(X) _Interlocked ## X ## 64
+# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
+# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
+
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_ATMC_T__ ethr_native_atomic64_t
+# define ETHR_AINT_T__ ethr_sint64_t
+
+# else
+# error "Unsupported integer size"
+# endif
-#ifndef ETHR_WIN_ATOMIC_COMMON__
-#define ETHR_WIN_ATOMIC_COMMON__
+typedef struct {
+ volatile ETHR_AINT_T__ value;
+} ETHR_ATMC_T__;
-#define ETHR_HAVE_NATIVE_ATOMICS 1
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
-# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
#else
-# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
#endif
-#if defined(_M_AMD64) || (defined(_M_IX86) \
- && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1
-#else
-# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0
-#endif
-/*
- * No configure test checking for interlocked acquire/release
- * versions have been written, yet. It should define
- * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and
- * only if, all used interlocked operations with barriers
- * exists.
- *
- * Note, that these are pure optimizations for the itanium
- * processor.
- */
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+ return (ETHR_AINT_T__ *) &var->value;
+}
-#include <intrin.h>
-#undef ETHR_COMPILER_BARRIER
-#define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
-#pragma intrinsic(_ReadWriteBarrier)
-#pragma intrinsic(_InterlockedCompareExchange)
-
-#if defined(_M_AMD64) || (defined(_M_IX86) \
- && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-#include <emmintrin.h>
-#include <mmintrin.h>
-#pragma intrinsic(_mm_mfence)
-#define ETHR_MEMORY_BARRIER _mm_mfence()
-#pragma intrinsic(_mm_sfence)
-#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence()
-#pragma intrinsic(_mm_lfence)
-#define ETHR_READ_MEMORY_BARRIER _mm_lfence()
-#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
#else
-
-#define ETHR_MEMORY_BARRIER \
-do { \
- volatile long x___ = 0; \
- _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \
-} while (0)
-
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
#endif
-#endif /* ETHR_WIN_ATOMIC_COMMON__ */
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+ var->value = i;
+}
#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
-#define ETHR_HAVE_NATIVE_ATOMIC32 1
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if defined(_M_IX86)
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+ else
+#endif /* _M_IX86 */
+ {
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ var->value = i;
+ }
+}
-/*
- * All used operations available as 32-bit intrinsics
- */
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
-#pragma intrinsic(_InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedExchangeAdd)
-#pragma intrinsic(_InterlockedExchange)
-#pragma intrinsic(_InterlockedAnd)
-#pragma intrinsic(_InterlockedOr)
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#pragma intrinsic(_InterlockedExchangeAdd_acq)
-#pragma intrinsic(_InterlockedIncrement_acq)
-#pragma intrinsic(_InterlockedDecrement_rel)
-#pragma intrinsic(_InterlockedCompareExchange_acq)
-#pragma intrinsic(_InterlockedCompareExchange_rel)
-#endif
+#if defined(ETHR_WIN_HAVE_XCHG)
-#define ETHR_ILCKD__(X) _Interlocked ## X
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
#else
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
#endif
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic32_t
-#define ETHR_AINT_T__ ethr_sint32_t
-
-#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+ (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+}
-#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#endif
-/*
- * _InterlockedCompareExchange64() is required. The other may not
- * be available, but if so, we can generate them.
- */
-#pragma intrinsic(_InterlockedCompareExchange64)
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
#else
-#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
#endif
-#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \
-{ \
- __int64 NEW, ACT, EXP; \
- ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \
- do { \
- EXP = ACT; \
- { OPS; } \
- ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \
- } while (ACT != EXP); \
- return RET; \
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+ return var->value;
}
-#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \
-static __forceinline __int64 \
-FUNC(__int64 volatile *ptr) \
-ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
-
-#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \
-static __forceinline __int64 \
-FUNC(__int64 volatile *ptr, __int64 ARG) \
-ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
+#if defined(ETHR_WIN_HAVE_XCHG_ADD)
-#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
-#pragma intrinsic(_InterlockedDecrement64)
-#else
-ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp,
- new = act - 1, new)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
-#pragma intrinsic(_InterlockedIncrement64)
-#else
-ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp,
- new = act + 1, new)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
-#pragma intrinsic(_InterlockedExchangeAdd64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp,
- new = act + arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
-#pragma intrinsic(_InterlockedExchange64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp,
- new = arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDAND64
-#pragma intrinsic(_InterlockedAnd64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp,
- new = act & arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDOR64
-#pragma intrinsic(_InterlockedOr64)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp,
- new = act | arg, arg, act)
-#endif
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#pragma intrinsic(_InterlockedExchangeAdd64_acq)
-#pragma intrinsic(_InterlockedIncrement64_acq)
-#pragma intrinsic(_InterlockedDecrement64_rel)
-#pragma intrinsic(_InterlockedCompareExchange64_acq)
-#pragma intrinsic(_InterlockedCompareExchange64_rel)
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
#endif
-#define ETHR_ILCKD__(X) _Interlocked ## X ## 64
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
-#else
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+ return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+}
+
#endif
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic64_t
-#define ETHR_AINT_T__ ethr_sint64_t
+#if defined(ETHR_WIN_HAVE_INC)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
#else
-#error "Unsupported integer size"
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
#endif
-typedef struct {
- volatile ETHR_AINT_T__ value;
-} ETHR_ATMC_T__;
-
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-
-static ETHR_INLINE ETHR_AINT_T__ *
-ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
{
- return (ETHR_AINT_T__ *) &var->value;
+ return ETHR_ILCKD__(Increment)(&var->value);
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
- var->value = i;
-#else
- (void) ETHR_ILCKD__(Exchange)(&var->value, i);
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
- var->value = i;
+#if defined(ETHR_WIN_HAVE_INC_ACQ)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
#else
- (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
{
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
- return var->value;
-#else
- return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
-#endif
+ return ETHR_ILCKD_ACQ__(Increment)(&var->value);
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
- (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr);
-}
+#endif
+
+#if defined(ETHR_WIN_HAVE_DEC)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
{
- return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+ return ETHR_ILCKD__(Decrement)(&var->value);
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
- (void) ETHR_ILCKD__(Increment)(&var->value);
-}
+#endif
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
- (void) ETHR_ILCKD__(Decrement)(&var->value);
-}
+#if defined(ETHR_WIN_HAVE_DEC_REL)
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
- return ETHR_ILCKD__(Increment)(&var->value);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
{
- return ETHR_ILCKD__(Decrement)(&var->value);
+ return ETHR_ILCKD_REL__(Decrement)(&var->value);
}
+#endif
+
+#if defined(ETHR_WIN_HAVE_AND)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
return ETHR_ILCKD__(And)(&var->value, mask);
}
+#endif
+
+#if defined(ETHR_WIN_HAVE_OR)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
return ETHR_ILCKD__(Or)(&var->value, mask);
}
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
- ETHR_AINT_T__ new,
- ETHR_AINT_T__ old)
-{
- return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
-}
+#endif
+#if defined(ETHR_WIN_HAVE_XCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
{
return ETHR_ILCKD__(Exchange)(&var->value, new);
}
-/*
- * Atomic ops with at least specified barriers.
- */
+#endif
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
- ETHR_AINT_T__ val = var->value;
- ETHR_COMPILER_BARRIER;
- return val;
+#if defined(ETHR_WIN_HAVE_CMPXCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
#else
- return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
#endif
-}
static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+ ETHR_AINT_T__ new,
+ ETHR_AINT_T__ old)
{
- return ETHR_ILCKD_ACQ__(Increment)(&var->value);
+ return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
- ETHR_COMPILER_BARRIER;
- var->value = i;
-#else
- (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i);
#endif
-}
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
-{
- (void) ETHR_ILCKD_REL__(Decrement)(&var->value);
-}
+#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ)
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
-{
- return ETHR_ILCKD_REL__(Decrement)(&var->value);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
@@ -391,6 +435,16 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
}
+#endif
+
+#if defined(ETHR_WIN_HAVE_CMPXCHG_REL)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
+#else
+# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
+#endif
+
static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
ETHR_AINT_T__ new,
@@ -399,6 +453,8 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
}
+#endif
+
#endif /* ETHR_TRY_INLINE_FUNCS */
#undef ETHR_ILCKD__
@@ -408,8 +464,17 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
#undef ETHR_ATMC_T__
#undef ETHR_AINT_T__
#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
-
-#endif /* _MSC_VER */
+#undef ETHR_WIN_HAVE_CMPXCHG
+#undef ETHR_WIN_HAVE_DEC
+#undef ETHR_WIN_HAVE_INC
+#undef ETHR_WIN_HAVE_XCHG_ADD
+#undef ETHR_WIN_HAVE_XCHG
+#undef ETHR_WIN_HAVE_AND
+#undef ETHR_WIN_HAVE_OR
+#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#undef ETHR_WIN_HAVE_INC_ACQ
+#undef ETHR_WIN_HAVE_DEC_REL
+#undef ETHR_WIN_HAVE_CMPXCHG_ACQ
+#undef ETHR_WIN_HAVE_CMPXCHG_REL
#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_dw_atomic.h b/erts/include/internal/win/ethr_dw_atomic.h
new file mode 100644
index 0000000000..a3e7ffc3aa
--- /dev/null
+++ b/erts/include/internal/win/ethr_dw_atomic.h
@@ -0,0 +1,154 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics for windows
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#ifndef ETHR_X86_DW_ATOMIC_H__
+# define ETHR_X86_DW_ATOMIC_H__
+# if ((ETHR_SIZEOF_PTR == 4 \
+ && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64)) \
+ || (ETHR_SIZEOF_PTR == 8 \
+ && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128)))
+# define ETHR_INCLUDE_DW_ATOMIC_IMPL__
+# endif
+#endif
+
+#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+
+# if ETHR_SIZEOF_PTR == 4
+# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+# else
+# define ETHR_HAVE_NATIVE_DW_ATOMIC
+# endif
+# define ETHR_NATIVE_DW_ATOMIC_IMPL "windows-interlocked"
+
+# if defined(_M_IX86) || defined(_M_AMD64)
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+ ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+# endif
+
+# include <intrin.h>
+# if ETHR_SIZEOF_PTR == 4
+# pragma intrinsic(_InterlockedCompareExchange64)
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+# else
+# pragma intrinsic(_InterlockedCompareExchange128)
+# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+# endif
+
+typedef volatile __int64 * ethr_native_dw_ptr_t;
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+ (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+#ifdef ETHR_NATIVE_SU_DW_SINT_T
+ volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+#endif
+ volatile ethr_sint_t sint[3];
+ volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#ifdef ETHR_DEBUG
+# define ETHR_DW_DBG_ALIGNED__(PTR) \
+ ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+#else
+# define ETHR_DW_DBG_ALIGNED__(PTR)
+#endif
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+ ethr_sint_t *p = (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+ ETHR_DW_DBG_ALIGNED__(p);
+ return p;
+}
+
+
+#if ETHR_SIZEOF_PTR == 4
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE ethr_sint64_t
+ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+ ethr_sint64_t new,
+ ethr_sint64_t exp)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ ETHR_DW_DBG_ALIGNED__(p);
+ return (ethr_sint64_t) _InterlockedCompareExchange64(p, new, exp);
+}
+
+#elif ETHR_SIZEOF_PTR == 8
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+
+#ifdef ETHR_BIGENDIAN
+# define ETHR_WIN_LOW_WORD__ 1
+# define ETHR_WIN_HIGH_WORD__ 0
+#else
+# define ETHR_WIN_LOW_WORD__ 0
+# define ETHR_WIN_HIGH_WORD__ 1
+#endif
+
+static ETHR_INLINE int
+ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+ ethr_sint_t *new,
+ ethr_sint_t *xchg)
+{
+ ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+ ETHR_DW_DBG_ALIGNED__(p);
+ return (int) _InterlockedCompareExchange128(p,
+ new[ETHR_WIN_HIGH_WORD__],
+ new[ETHR_WIN_LOW_WORD__],
+ xchg);
+}
+
+#endif
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h
index 598816b2c6..6363174a74 100644
--- a/erts/include/internal/win/ethr_event.h
+++ b/erts/include/internal/win/ethr_event.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -21,12 +21,12 @@
* Author: Rickard Green
*/
-#define ETHR_EVENT_OFF_WAITER__ ((long) -1)
-#define ETHR_EVENT_OFF__ ((long) 1)
-#define ETHR_EVENT_ON__ ((long) 0)
+#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1)
+#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1)
+#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0)
typedef struct {
- volatile long state;
+ ethr_atomic32_t state;
HANDLE handle;
} ethr_event;
@@ -38,7 +38,7 @@ static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
{
/* _InterlockedExchange() imply a full memory barrier which is important */
- long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__);
+ ethr_sint32_t state = ethr_atomic32_xchg_wb(&e->state, ETHR_EVENT_ON__);
if (state == ETHR_EVENT_OFF_WAITER__) {
if (!SetEvent(e->handle))
ETHR_FATAL_ERROR__(ethr_win_get_errno__());
@@ -48,8 +48,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
{
- /* _InterlockedExchange() imply a full memory barrier which is important */
- InterlockedExchange(&e->state, ETHR_EVENT_OFF__);
+ ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__);
+ ETHR_MEMORY_BARRIER;
}
#endif
diff --git a/erts/include/internal/win/ethr_membar.h b/erts/include/internal/win/ethr_membar.h
new file mode 100644
index 0000000000..8237660b2c
--- /dev/null
+++ b/erts/include/internal/win/ethr_membar.h
@@ -0,0 +1,145 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for Windows
+ * Author: Rickard Green
+ */
+
+#if (!defined(ETHR_WIN_MEMBAR_H__) \
+ && (defined(_MSC_VER) && _MSC_VER >= 1400) \
+ && (defined(_M_AMD64) \
+ || defined(_M_IA64) \
+ || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE)))
+#define ETHR_WIN_MEMBAR_H__
+
+#define ETHR_LoadLoad (1 << 0)
+#define ETHR_LoadStore (1 << 1)
+#define ETHR_StoreLoad (1 << 2)
+#define ETHR_StoreStore (1 << 3)
+
+#include <intrin.h>
+#undef ETHR_COMPILER_BARRIER
+#define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#pragma intrinsic(_ReadWriteBarrier)
+
+#pragma intrinsic(_InterlockedCompareExchange)
+
+#define ETHR_MB_USING_INTERLOCKED__ \
+do { \
+ volatile long x___ = 0; \
+ (void) _InterlockedCompareExchange(&x___, 2, 1); \
+} while (0)
+
+#if defined(_M_IA64)
+
+#define ETHR_MEMBAR(B) __mf()
+
+#elif defined(_M_AMD64) || defined(_M_IX86)
+
+#include <emmintrin.h>
+#include <mmintrin.h>
+
+#if ETHR_SIZEOF_PTR == 4
+# define ETHR_NO_SSE2_MB__ ETHR_MB_USING_INTERLOCKED__
+#endif
+#pragma intrinsic(_mm_mfence)
+#pragma intrinsic(_mm_sfence)
+#pragma intrinsic(_mm_lfence)
+
+static __forceinline void
+ethr_cfence__(void)
+{
+ _ReadWriteBarrier();
+}
+
+static __forceinline void
+ethr_mfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MB__;
+ else
+#endif
+ _mm_mfence();
+}
+
+static __forceinline void
+ethr_sfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MB__;
+ else
+#endif
+ _mm_sfence();
+}
+
+static __forceinline void
+ethr_lfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+ if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+ ETHR_NO_SSE2_MB__;
+ else
+#endif
+ _mm_lfence();
+}
+
+#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \
+ ethr_sfence__(), \
+ ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \
+ ethr_lfence__(), \
+ ethr_mfence__()))
+
+#ifdef ETHR_X86_OUT_OF_ORDER
+
+#define ETHR_MEMBAR(B) \
+ ETHR_X86_OUT_OF_ORDER_MEMBAR((B))
+
+#else /* !ETHR_X86_OUT_OF_ORDER (the default) */
+
+/*
+ * We assume that only stores before loads may be reordered. That is,
+ * we assume that *no* instructions like these are used:
+ * - CLFLUSH,
+ * - streaming stores executed with non-temporal move,
+ * - string operations, or
+ * - other instructions which aren't LoadLoad, LoadStore, and StoreStore
+ * ordered by themselves
+ * If such instructions are used, either insert memory barriers
+ * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or
+ * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32
+ * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2.
+ */
+
+#define ETHR_MEMBAR(B) \
+ ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__())
+
+#endif
+
+#else /* No knowledge about platform; use interlocked fallback */
+
+#define ETHR_MEMBAR(B) ETHR_MB_USING_INTERLOCKED__
+#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MB_USING_INTERLOCKED__
+
+#endif
+
+#endif /* ETHR_WIN_MEMBAR_H__ */
diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h
index c01b17cf14..8be35e810e 100644
--- a/erts/include/internal/win/ethread.h
+++ b/erts/include/internal/win/ethread.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,11 +25,13 @@
#ifndef ETHREAD_WIN_H__
#define ETHREAD_WIN_H__
+#include "ethr_membar.h"
#define ETHR_ATOMIC_WANT_32BIT_IMPL__
#include "ethr_atomic.h"
#if ETHR_SIZEOF_PTR == 8
# define ETHR_ATOMIC_WANT_64BIT_IMPL__
# include "ethr_atomic.h"
#endif
+#include "ethr_dw_atomic.h"
#endif
diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in
index 757b3b24e2..12b8732735 100644
--- a/erts/lib_src/Makefile.in
+++ b/erts/lib_src/Makefile.in
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 2004-2010. All Rights Reserved.
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -65,7 +65,7 @@ TYPE_SUFFIX=.purecov
PRE_LD=purecov $(PURECOV_BUILD_OPTIONS)
else
ifeq ($(TYPE),gcov)
-CFLAGS=@DEBUG_CFLAGS@ -fprofile-arcs -ftest-coverage -O0
+CFLAGS=@DEBUG_CFLAGS@ -DGCOV -fprofile-arcs -ftest-coverage -O0
TYPE_SUFFIX=.gcov
PRE_LD=
else
@@ -288,6 +288,10 @@ ETHREAD_LIB_SRC=common/ethr_aux.c \
common/ethr_cbf.c \
$(ETHR_THR_LIB_BASE_DIR)/ethread.c \
$(ETHR_THR_LIB_BASE_DIR)/ethr_event.c
+ETHR_X86_SSE2_ASM=@ETHR_X86_SSE2_ASM@
+ifeq ($(ETHR_X86_SSE2_ASM),yes)
+ETHREAD_LIB_SRC += pthread/ethr_x86_sse2_asm.c
+endif
ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX)
ifeq ($(USING_VC),yes)
@@ -382,10 +386,8 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS)
# Object files
#
-ifeq ($(TYPE)-@GCC@,debug-yes)
-$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c
- $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@
-endif
+$(r_OBJ_DIR)/ethr_x86_sse2_asm.o: pthread/ethr_x86_sse2_asm.c
+ $(CC) -msse2 $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
$(r_OBJ_DIR)/%.o: common/%.c
$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
index 94557d904a..5796bdc22e 100644
--- a/erts/lib_src/common/ethr_atomics.c
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -1,7 +1,16 @@
/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -18,385 +27,4329 @@
*/
/*
- * Description: The ethread atomic API
+ * Description: The ethread atomics API
* Author: Rickard Green
*/
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+ * - cmpxchg
+ * - xchg
+ * - set
+ * - init
+ * - add_read
+ * - read
+ * - inc_read
+ * - dec_read
+ * - add
+ * - inc
+ * - dec
+ * - read_band
+ * - read_bor
+ *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+ * - cmpxchg
+ * - set
+ * - read
+ * - init
+ *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * memory barrier semantics:
+ * - rb (read barrier)
+ * - wb (write barrier)
+ * - acqb (acquire barrier)
+ * - relb (release barrier)
+ * - mb (full memory barrier)
+ *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ * ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ * ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ * ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ * and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ * either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ * (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ * A macro informing about the presence of the native
+ * implementation:
+ *
+ * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ * A macro naming (a string constant) the implementation:
+ *
+ * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ * Each implemented native atomic function has to
+ * be accompanied by a defined macro on the following
+ * form informing about its presence:
+ *
+ * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ * A (sparc-v9 style) membar macro:
+ *
+ * - ETHR_MEMBAR(B)
+ *
+ * Which takes a combination of the following macros
+ * or:ed (using |) together:
+ *
+ * - ETHR_LoadLoad
+ * - ETHR_LoadStore
+ * - ETHR_StoreLoad
+ * - ETHR_StoreStore
+ *
+ */
+
+
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_TRY_INLINE_FUNCS
+#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
#define ETHR_ATOMIC_IMPL__
#include "ethread.h"
#include "ethr_internal.h"
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+/*
+ * Spinlock based fallback for atomics used in absence of a native
+ * implementation.
+ */
+
+#define ETHR_ATMC_FLLBK_ADDR_BITS 10
+#define ETHR_ATMC_FLLBK_ADDR_SHIFT 6
+
+typedef struct {
+ union {
+ ethr_spinlock_t lck;
+ char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
+ } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \
+(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \
+ & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \
+do { \
+ ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \
+ ethr_spin_lock(slp__); \
+ { EXPS; } \
+ ethr_spin_unlock(slp__); \
+} while (0)
+
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#endif
+
+
+#if defined(ETHR_AMC_FALLBACK__)
+
+/*
+ * Fallback for large sized (word and/or double word size) atomics using
+ * an "Atomic Modification Counter" based on smaller sized native atomics.
+ *
+ * We use a 63-bit modification counter and a one bit exclusive flag.
+ * If 32-bit native atomics are used, we need two 32-bit native atomics.
+ * The exclusive flag is the least significant bit, or if multiple atomics
+ * are used, the least significant bit of the least significant atomic.
+ *
+ * When using the AMC fallback the following is true:
+ * - Reads of the same atomic variable can be done in parallel.
+ * - Uncontended reads doesn't cause any cache line invalidations,
+ * since no modifications are done.
+ * - Assuming that the AMC atomic(s) and the integer(s) containing the
+ * value of the implemented atomic resides in the same cache line,
+ * modifications will only cause invalidations of one cache line.
+ *
+ * When using the spinlock based fallback none of the above is true,
+ * however, the spinlock based fallback consumes less memory.
+ */
+
+# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2
+# error "Not supported"
+# endif
+# define ETHR_AMC_MAX_TRY_READ__ 10
+# ifdef ETHR_DEBUG
+# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \
+do { \
+ ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \
+ ETHR_ASSERT(act == (S) + 1); \
+ ETHR_ASSERT(act & 1); \
+} while (0)
+# else
+# define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
+# endif
+
+static ETHR_INLINE void
+amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ avar[0] = val[0];
+ if (dw)
+ avar[1] = val[1];
+#if ETHR_AMC_NO_ATMCS__ == 2
+ ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
#endif
+ ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
+}
+
+static ETHR_INLINE ETHR_AMC_SINT_T__
+amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
+{
+ ETHR_AMC_SINT_T__ state0 = prev_state0;
+ /* Set exclusive flag. */
+ while (1) {
+ ETHR_AMC_SINT_T__ act_state0, new_state0;
+ while (state0 & 1) { /* Wait until exclusive bit has been cleared */
+ ETHR_SPIN_BODY;
+ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+ }
+ /* Try to set exclusive bit */
+ new_state0 = state0 + 1;
+ act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
+ new_state0,
+ state0);
+ if (state0 == act_state0)
+ return state0; /* old state0 */
+ state0 = act_state0;
+ }
+}
+
+static ETHR_INLINE void
+amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+ ETHR_AMC_SINT_T__ state0 = old_state0;
+
+ /* Increment modification counter and reset exclusive flag. */
+
+ ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+ state0 += 2;
+
+ ETHR_ASSERT((state0 & 1) == 0);
+
+#if ETHR_AMC_NO_ATMCS__ == 2
+ if (state0 == 0) {
+ /*
+ * state0 wrapped, so we need to increment state1. There is no need
+ * for atomic inc op, since this is always done while having exclusive
+ * flag.
+ */
+ ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
+ state1++;
+ ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
+ }
+#endif
+ ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
+}
+
+static ETHR_INLINE void
+amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+ ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
+ /*
+ * Reset exclusive flag, but leave modification counter unchanged,
+ * i.e., restore state to what it was before setting exclusive
+ * flag.
+ */
+ ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
+}
+
+static ETHR_INLINE void
+amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+
+ state0 = amc_set_excl(amc, state0);
+
+ avar[0] = val[0];
+ if (dw)
+ avar[1] = val[1];
+
+ amc_inc_mc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+ ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
+{
+ /* *state0p should contain last read value if aborting */
+ ETHR_AMC_SINT_T__ old_state0;
+#if ETHR_AMC_NO_ATMCS__ == 2
+ ETHR_AMC_SINT_T__ state1;
+ int abrt;
+#endif
+
+ *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+ if ((*state0p) & 1)
+ return 0; /* exclusive flag set; abort */
+#if ETHR_AMC_NO_ATMCS__ == 2
+ state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
+#else
+ ETHR_COMPILER_BARRIER;
+#endif
+
+ val[0] = avar[0];
+ if (dw)
+ val[1] = avar[1];
+
+ ETHR_READ_MEMORY_BARRIER;
+
+ /*
+ * Abort if state has changed (i.e, either the exclusive
+ * flag is set, or modification counter changed).
+ */
+ old_state0 = *state0p;
+#if ETHR_AMC_NO_ATMCS__ == 2
+ *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+ abrt = (old_state0 != *state0p);
+ abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
+ return abrt == 0;
+#else
+ *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+ return old_state0 == *state0p;
+#endif
+}
+
+static ETHR_INLINE void
+amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ ETHR_AMC_SINT_T__ state0;
+ int i;
+
+#if ETHR_AMC_MAX_TRY_READ__ == 0
+ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+#else
+ for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
+ if (amc_try_read(amc, dw, avar, val, &state0))
+ return; /* read success */
+ ETHR_SPIN_BODY;
+ }
+#endif
+
+ state0 = amc_set_excl(amc, state0);
+
+ val[0] = avar[0];
+ if (dw)
+ val[1] = avar[1];
+
+ amc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+ ethr_sint_t *new, ethr_sint_t *xchg)
+{
+ ethr_sint_t val[2];
+ ETHR_AMC_SINT_T__ state0;
+
+ if (amc_try_read(amc, dw, avar, val, &state0)) {
+ if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
+ xchg[0] = val[0];
+ if (dw)
+ xchg[1] = val[1];
+ return 0; /* failed */
+ }
+ /* Operation will succeed if not interrupted */
+ }
+
+ state0 = amc_set_excl(amc, state0);
+
+ if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
+ xchg[0] = avar[0];
+ if (dw)
+ xchg[1] = avar[1];
+
+ ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+ amc_unset_excl(amc, state0);
+ return 0; /* failed */
+ }
+
+ avar[0] = new[0];
+ if (dw)
+ avar[1] = new[1];
+
+ amc_inc_mc_unset_excl(amc, state0);
+ return 1;
+}
+
+
+#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \
+do { \
+ ETHR_AMC_SINT_T__ state0__; \
+ state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \
+ state0__ = amc_set_excl((AMC), state0__); \
+ { OPS; } \
+ amc_inc_mc_unset_excl((AMC), state0__); \
+} while (0)
+
+#endif /* amc fallback */
+
int
ethr_init_atomics(void)
{
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
- {
- int i;
- for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
- int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
- if (res != 0)
- return res;
- }
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+ int i;
+ for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
+ int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+ if (res != 0)
+ return res;
}
#endif
return 0;
}
+
+/* ---------- Double word size atomic implementation ---------- */
+
+
+
/*
- * --- Pointer size atomics ---------------------------------------------------
+ * Double word atomics need runtime test.
*/
-ethr_sint_t *
-ethr_atomic_addr(ethr_atomic_t *var)
+int ethr_have_native_dw_atomic(void)
+{
+ return ethr_have_native_dw_atomic__();
+}
+
+
+/* --- addr() --- */
+
+ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var)
+{
+ ethr_sint_t *res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_addr__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = (ethr_sint_t *) ((&var->fallback))->sint;
+#else
+ res = (ethr_sint_t *) (&var->fallback);
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_addr__(var);
+}
+#endif
+
+
+/* -- cmpxchg() -- */
+
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ int res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ res = ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback),
+ {
+ res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+ if (res) {
+ (&var->fallback)->sint[0] = val->sint[0];
+ (&var->fallback)->sint[1] = val->sint[1];
+ }
+ else {
+ old_val->sint[0] = (&var->fallback)->sint[0];
+ old_val->sint[1] = (&var->fallback)->sint[1];
+ }
+ });
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
+}
+#endif
+
+
+/* -- set() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_set_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_set_mb__(var, val);
+}
+#endif
+
+
+/* -- read() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_read_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_read_mb__(var, val);
+}
+#endif
+
+
+/* -- init() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ ethr_dw_atomic_init_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+ ETHR_ASSERT(var);
+ ethr_dw_atomic_init_mb__(var, val);
+}
+#endif
+
+
+/* ---------- Word size atomic implementation ---------- */
+
+
+
+
+/* --- addr() --- */
+
+ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var)
+{
+ ethr_sint_t *res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_addr__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = (ethr_sint_t *) (var)->sint;
+#else
+ res = (ethr_sint_t *) var;
+#endif
+ return res;
+}
+
+
+/* -- cmpxchg() -- */
+
+
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg_rb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg_wb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg_acqb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg_relb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_cmpxchg_mb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ res = old_val;
+ (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- xchg() -- */
+
+
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_xchg_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- set() -- */
+
+
+void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_set(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_set(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ amc_set(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_set_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ amc_set(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- init() -- */
+
+
+void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_init(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_init(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ amc_init(&var->amc, 0, &var->sint, &val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_init_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ amc_init(&var->amc, 0, &var->sint, &val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- add_read() -- */
+
+
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_add_read_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- read() -- */
+
+
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&var->amc, 0, &var->sint, &res);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&var->amc, 0, &var->sint, &res);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ amc_read(&var->amc, 0, &var->sint, &res);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ amc_read(&var->amc, 0, &var->sint, &res);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ amc_read(&var->amc, 0, &var->sint, &res);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ amc_read(&var->amc, 0, &var->sint, &res);
+ ETHR_MEMBAR(ETHR_LoadStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+ return res;
+}
+
+
+/* -- inc_read() -- */
+
+
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_inc_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- dec_read() -- */
+
+
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_addr__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
}
-void
-ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i)
+ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic_init__(var, i);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
}
-void
-ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i)
+ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic_set__(var, i);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
}
-ethr_sint_t
-ethr_atomic_read(ethr_atomic_t *var)
+ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_read__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
}
-ethr_sint_t
-ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr)
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_add_read__(var, incr);
-}
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
+}
-ethr_sint_t
-ethr_atomic_inc_read(ethr_atomic_t *var)
+ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_inc_read__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_dec_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
}
-ethr_sint_t
-ethr_atomic_dec_read(ethr_atomic_t *var)
+
+/* -- add() -- */
+
+
+void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_dec_read__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
}
-void
-ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr)
+void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic_add__(var, incr);
-}
-
-void
-ethr_atomic_inc(ethr_atomic_t *var)
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_add_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- inc() -- */
+
+
+void ethr_atomic_inc(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
ethr_atomic_inc__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
}
-void
-ethr_atomic_dec(ethr_atomic_t *var)
+void ethr_atomic_inc_rb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic_dec__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_inc_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
}
-ethr_sint_t
-ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask)
+void ethr_atomic_inc_wb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_read_band__(var, mask);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_inc_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
}
-ethr_sint_t
-ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask)
+void ethr_atomic_inc_acqb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_read_bor__(var, mask);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_inc_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-ethr_sint_t
-ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new)
+void ethr_atomic_inc_relb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_xchg__(var, new);
-}
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_inc_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
-ethr_sint_t
-ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected)
+void ethr_atomic_inc_mb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg__(var, new, expected);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_inc_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-ethr_sint_t
-ethr_atomic_read_acqb(ethr_atomic_t *var)
+
+/* -- dec() -- */
+
+
+void ethr_atomic_dec(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_read_acqb__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_dec__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
}
-ethr_sint_t
-ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+void ethr_atomic_dec_rb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_inc_read_acqb__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_dec_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
}
-void
-ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i)
+void ethr_atomic_dec_wb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic_set_relb__(var, i);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_dec_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
}
-void
-ethr_atomic_dec_relb(ethr_atomic_t *var)
+void ethr_atomic_dec_acqb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_dec_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
ethr_atomic_dec_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
}
-ethr_sint_t
-ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+void ethr_atomic_dec_mb(ethr_atomic_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_dec_read_relb__(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ ethr_atomic_dec_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-ethr_sint_t
-ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+
+/* -- read_band() -- */
+
+
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg_acqb__(var, new, exp);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
}
-ethr_sint_t
-ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
{
+ ethr_sint_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg_relb__(var, new, exp);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
}
+ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_band_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- read_bor() -- */
+
+
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+ ethr_sint_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic_read_bor_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* ---------- 32-bit atomic implementation ---------- */
+
+
+
+
+/* --- addr() --- */
+
+ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var)
+{
+ ethr_sint32_t *res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_addr__(var);
+
+#else
+ res = (ethr_sint32_t *) var;
+#endif
+ return res;
+}
+
+
+/* -- cmpxchg() -- */
+
+
+ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg__(var, val, old_val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg_rb__(var, val, old_val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg_wb__(var, val, old_val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg_acqb__(var, val, old_val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg_relb__(var, val, old_val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_cmpxchg_mb__(var, val, old_val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- xchg() -- */
+
+
+ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_xchg_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- set() -- */
+
+
+void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_set_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- init() -- */
+
+
+void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_init_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- add_read() -- */
+
+
+ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_add_read_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- read() -- */
+
+
+ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_rb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_wb__(var);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_acqb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_relb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_mb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+ return res;
+}
+
+
+/* -- inc_read() -- */
+
+
+ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read_rb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read_wb__(var);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read_acqb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read_relb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_inc_read_mb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- dec_read() -- */
-/*
- * --- 32-bit atomics ---------------------------------------------------------
- */
-ethr_sint32_t *
-ethr_atomic32_addr(ethr_atomic32_t *var)
+ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_addr__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
}
-void
-ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i)
+ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic32_init__(var, i);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read_rb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
}
-void
-ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i)
+ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic32_set__(var, i);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read_wb__(var);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
}
-ethr_sint32_t
-ethr_atomic32_read(ethr_atomic32_t *var)
+ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_read__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read_acqb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
}
+ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read_relb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+ return res;
+}
-ethr_sint32_t
-ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr)
+ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_add_read__(var, incr);
-}
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_dec_read_mb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- add() -- */
-ethr_sint32_t
-ethr_atomic32_inc_read(ethr_atomic32_t *var)
+
+void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_inc_read__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_dec_read(ethr_atomic32_t *var)
+void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_dec_read__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
}
-void
-ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr)
+void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic32_add__(var, incr);
-}
-
-void
-ethr_atomic32_inc(ethr_atomic32_t *var)
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_add_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- inc() -- */
+
+
+void ethr_atomic32_inc(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
ethr_atomic32_inc__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
}
-void
-ethr_atomic32_dec(ethr_atomic32_t *var)
+void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic32_dec__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_inc_rb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask)
+void ethr_atomic32_inc_wb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_read_band__(var, mask);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_inc_wb__(var);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask)
+void ethr_atomic32_inc_acqb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_read_bor__(var, mask);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_inc_acqb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new)
+void ethr_atomic32_inc_relb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_xchg__(var, new);
-}
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_inc_relb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
-ethr_sint32_t
-ethr_atomic32_cmpxchg(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t expected)
+}
+
+void ethr_atomic32_inc_mb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_cmpxchg__(var, new, expected);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_inc_mb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_read_acqb(ethr_atomic32_t *var)
+
+/* -- dec() -- */
+
+
+void ethr_atomic32_dec(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_read_acqb__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_dec__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_dec_rb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
}
-ethr_sint32_t
-ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
+void ethr_atomic32_dec_wb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_inc_read_acqb__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_dec_wb__(var);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
}
-void
-ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i)
+void ethr_atomic32_dec_acqb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- ethr_atomic32_set_relb__(var, i);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_dec_acqb__(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
}
-void
-ethr_atomic32_dec_relb(ethr_atomic32_t *var)
+void ethr_atomic32_dec_relb(ethr_atomic32_t *var)
{
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
ethr_atomic32_dec_relb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic32_dec_mb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ ethr_atomic32_dec_mb__(var);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- read_band() -- */
+
+
+ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_band_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+/* -- read_bor() -- */
+
+
+ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor_rb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+ return res;
}
-ethr_sint32_t
-ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
+ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_dec_read_relb__(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor_wb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_StoreStore);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
}
-ethr_sint32_t
-ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
+ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_cmpxchg_acqb__(var, new, exp);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor_acqb__(var, val);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
}
-ethr_sint32_t
-ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var,
- ethr_sint32_t new,
- ethr_sint32_t exp)
+ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
+ ethr_sint32_t res;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(var);
- return ethr_atomic32_cmpxchg_relb__(var, new, exp);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor_relb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+ return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+ ethr_sint32_t res;
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+ res = ethr_atomic32_read_bor_mb__(var, val);
+#else
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+ ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+ return res;
+}
+
+
+
+/* --------- Info functions --------- */
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+char *zero_ops[] = {NULL};
+#endif
+
+
+static char *native_su_dw_atomic_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
+ "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
+ "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
+ "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
+ "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
+ "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+ "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+ "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
+ "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
+ "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
+ "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
+ "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
+ "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+ "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
+ "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
+ "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
+ "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
+ "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
+ "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
+ "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
+ "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
+ "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
+ "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
+ "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
+ "init_mb",
+#endif
+ NULL
+};
+
+char **
+ethr_native_su_dw_atomic_ops(void)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ return &zero_ops[0];
+#endif
+ return &native_su_dw_atomic_ops[0];
+}
+
+
+static char *native_dw_atomic_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
+ "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
+ "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
+ "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
+ "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
+ "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+ "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
+ "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
+ "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
+ "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
+ "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
+ "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
+ "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
+ "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
+ "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
+ "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
+ "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
+ "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
+ "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
+ "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
+ "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
+ "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
+ "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
+ "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
+ "init_mb",
+#endif
+ NULL
+};
+
+char **
+ethr_native_dw_atomic_ops(void)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ return &zero_ops[0];
+#endif
+ return &native_dw_atomic_ops[0];
+}
+
+
+static char *native_atomic64_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+ "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+ "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+ "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+ "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+ "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+ "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+ "xchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+ "xchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+ "xchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+ "xchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+ "xchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+ "xchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+ "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+ "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+ "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+ "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+ "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+ "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+ "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+ "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+ "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+ "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+ "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+ "init_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+ "add_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+ "add_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+ "add_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+ "add_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+ "add_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+ "add_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+ "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+ "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+ "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+ "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+ "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+ "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+ "inc_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+ "inc_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+ "inc_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+ "inc_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+ "inc_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+ "inc_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+ "dec_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+ "dec_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+ "dec_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+ "dec_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+ "dec_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+ "dec_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+ "add",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+ "add_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+ "add_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+ "add_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+ "add_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+ "add_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+ "inc",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+ "inc_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+ "inc_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+ "inc_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+ "inc_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+ "inc_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+ "dec",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+ "dec_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+ "dec_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+ "dec_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+ "dec_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+ "dec_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+ "and_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+ "and_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+ "and_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+ "and_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+ "and_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+ "and_retold_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+ "or_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+ "or_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+ "or_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+ "or_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+ "or_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+ "or_retold_mb",
+#endif
+ NULL
+};
+
+char **
+ethr_native_atomic64_ops(void)
+{
+
+ return &native_atomic64_ops[0];
}
+
+static char *native_atomic32_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+ "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+ "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+ "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+ "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+ "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+ "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+ "xchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+ "xchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+ "xchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+ "xchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+ "xchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+ "xchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+ "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+ "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+ "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+ "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+ "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+ "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+ "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+ "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+ "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+ "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+ "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+ "init_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+ "add_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+ "add_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+ "add_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+ "add_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+ "add_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+ "add_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+ "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+ "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+ "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+ "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+ "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+ "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+ "inc_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+ "inc_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+ "inc_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+ "inc_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+ "inc_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+ "inc_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+ "dec_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+ "dec_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+ "dec_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+ "dec_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+ "dec_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+ "dec_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+ "add",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+ "add_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+ "add_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+ "add_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+ "add_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+ "add_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+ "inc",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+ "inc_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+ "inc_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+ "inc_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+ "inc_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+ "inc_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+ "dec",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+ "dec_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+ "dec_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+ "dec_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+ "dec_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+ "dec_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+ "and_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+ "and_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+ "and_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+ "and_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+ "and_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+ "and_retold_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+ "or_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+ "or_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+ "or_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+ "or_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+ "or_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+ "or_retold_mb",
+#endif
+ NULL
+};
+
+char **
+ethr_native_atomic32_ops(void)
+{
+
+ return &native_atomic32_ops[0];
+}
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
index 2c3e25a805..521640317e 100644
--- a/erts/lib_src/common/ethr_aux.c
+++ b/erts/lib_src/common/ethr_aux.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -31,10 +31,6 @@
#define ETHR_INLINE_FUNC_NAME_(X) X ## __
#define ETHR_AUX_IMPL__
-#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in
- native atomic implementations
- for optimized fallbacks of
- spinlocks and rwspinlocks */
#include "ethread.h"
#include "ethr_internal.h"
#include <string.h>
@@ -75,10 +71,87 @@ static int main_threads;
static int init_ts_event_alloc(void);
+ethr_runtime_t ethr_runtime__
+#ifdef __GNUC__
+__attribute__ ((aligned (ETHR_CACHE_LINE_SIZE)))
+#endif
+ ;
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+/*
+ * x86/x86_64 specifics shared between windows and
+ * pthread implementations.
+ */
+
+#define ETHR_IS_X86_VENDOR(V, B, C, D) \
+ (sizeof(V) == 13 && is_x86_vendor((V), (B), (C), (D)))
+
+static ETHR_INLINE int
+is_x86_vendor(char *str, int ebx, int ecx, int edx)
+{
+ return (*((int *) &str[0]) == ebx
+ && *((int *) &str[sizeof(int)]) == edx
+ && *((int *) &str[sizeof(int)*2]) == ecx);
+}
+
+static void
+x86_init(void)
+{
+ int eax, ebx, ecx, edx;
+
+ eax = ebx = ecx = edx = 0;
+
+ ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx);
+
+ if (eax > 0
+ && (ETHR_IS_X86_VENDOR("GenuineIntel", ebx, ecx, edx)
+ || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx))) {
+ eax = 1;
+ ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx);
+ }
+ else {
+ /*
+ * The meaning of the feature flags for this
+ * vendor have not been verified.
+ */
+ eax = ebx = ecx = edx = 0;
+ }
+
+ /*
+ * The feature flags tested below have only been verified
+ * for vendors checked above. Also note that only these
+ * feature flags have been verified to have these specific
+ * meanings. If another feature flag test is introduced,
+ * it has to be verified to have the same meaning for all
+ * vendors above.
+ */
+
+#if ETHR_SIZEOF_PTR == 8
+ /* bit 13 of ecx is set if we have cmpxchg16b */
+ ethr_runtime__.conf.have_dw_cmpxchg = (ecx & (1 << 13));
+#elif ETHR_SIZEOF_PTR == 4
+ /* bit 8 of edx is set if we have cmpxchg8b */
+ ethr_runtime__.conf.have_dw_cmpxchg = (edx & (1 << 8));
+#else
+# error "Not supported"
+#endif
+ /* bit 26 of edx is set if we have sse2 */
+ ethr_runtime__.conf.have_sse2 = (edx & (1 << 26));
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
+
int
ethr_init_common__(ethr_init_data *id)
{
int res;
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+ x86_init();
+#endif
+
if (id) {
ethr_thr_prepare_func__ = id->thread_create_prepare_func;
ethr_thr_parent_func__ = id->thread_create_parent_func;
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index 2ddef32dfc..81fd6af80a 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -26,8 +26,9 @@
#include "config.h"
#endif
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_MTX_FUNC_NAME_(X) X ## __
#define ETHR_MUTEX_IMPL__
+#define ETHR_TRY_INLINE_FUNCS
#include <limits.h>
#include "ethread.h"
diff --git a/erts/lib_src/pthread/ethr_x86_sse2_asm.c b/erts/lib_src/pthread/ethr_x86_sse2_asm.c
new file mode 100644
index 0000000000..6cbe73cf16
--- /dev/null
+++ b/erts/lib_src/pthread/ethr_x86_sse2_asm.c
@@ -0,0 +1,31 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: sse2 asm:s
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+/* ETHR_X86_SSE2_ASM_C__ will trigger asm:s to compile to be included */
+#define ETHR_X86_SSE2_ASM_C__
+#include "ethread.h"
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c
index f047104103..ad29249bac 100644
--- a/erts/lib_src/pthread/ethread.c
+++ b/erts/lib_src/pthread/ethread.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -121,6 +121,98 @@ ethr_ts_event *ethr_get_tse__(void)
return pthread_getspecific(ethr_ts_event_key__);
}
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+
+static volatile int lwsync_caused_sigill;
+
+static void
+handle_lwsync_sigill(int signum)
+{
+ lwsync_caused_sigill = 1;
+}
+
+static int
+ppc_init__(void)
+{
+ struct sigaction act, oact;
+ lwsync_caused_sigill = 0;
+
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = 0;
+ act.sa_handler = handle_lwsync_sigill;
+ if (sigaction(SIGILL, &act, &oact) != 0)
+ return errno;
+
+ __asm__ __volatile__ ("lwsync\n\t" : : : "memory");
+
+ act.sa_flags = 0;
+ act.sa_handler = SIG_DFL;
+ if (sigaction(SIGILL, &act, &oact) != 0)
+ return errno;
+
+ ethr_runtime__.conf.have_lwsync = (int) !lwsync_caused_sigill;
+ return 0;
+}
+
+#endif
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+void
+ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx)
+{
+#if ETHR_SIZEOF_PTR == 4
+ int have_cpuid;
+ /*
+ * If it is possible to toggle eflags bit 21,
+ * we have the cpuid instruction.
+ */
+ __asm__ ("pushf\n\t"
+ "popl %%eax\n\t"
+ "movl %%eax, %%ecx\n\t"
+ "xorl $0x200000, %%eax\n\t"
+ "pushl %%eax\n\t"
+ "popf\n\t"
+ "pushf\n\t"
+ "popl %%eax\n\t"
+ "movl $0x0, %0\n\t"
+ "xorl %%ecx, %%eax\n\t"
+ "jz no_cpuid\n\t"
+ "movl $0x1, %0\n\t"
+ "no_cpuid:\n\t"
+ : "=r"(have_cpuid)
+ :
+ : "%eax", "%ecx", "cc");
+ if (!have_cpuid) {
+ *eax = *ebx = *ecx = *edx = 0;
+ return;
+ }
+#endif
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ /*
+ * When position independet code is used in 32-bit mode, the B register
+ * is used for storage of global offset table address, and we may not
+ * use it as input or output in an asm. We need to save and restore the
+ * B register explicitly (for some reason gcc doesn't provide this
+ * service to us).
+ */
+ __asm__ ("pushl %%ebx\n\t"
+ "cpuid\n\t"
+ "movl %%ebx, %1\n\t"
+ "popl %%ebx\n\t"
+ : "=a"(*eax), "=r"(*ebx), "=c"(*ecx), "=d"(*edx)
+ : "0"(*eax)
+ : "cc");
+#else
+ __asm__ ("cpuid\n\t"
+ : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx)
+ : "0"(*eax)
+ : "cc");
+#endif
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
/*
* --------------------------------------------------------------------------
* Exported functions
@@ -137,6 +229,12 @@ ethr_init(ethr_init_data *id)
ethr_not_inited__ = 0;
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+ res = ppc_init__();
+ if (res != 0)
+ goto error;
+#endif
+
res = ethr_init_common__(id);
if (res != 0)
goto error;
@@ -146,6 +244,8 @@ ethr_init(ethr_init_data *id)
child_wait_spin_count = 0;
res = pthread_key_create(&ethr_ts_event_key__, ethr_ts_event_destructor__);
+ if (res != 0)
+ goto error;
return 0;
error:
diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api
new file mode 100755
index 0000000000..f4e71c7618
--- /dev/null
+++ b/erts/lib_src/utils/make_atomics_api
@@ -0,0 +1,2186 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-mode(compile).
+
+%%%-------------------------------------------------------------------
+%%% @author Rickard Green <[email protected]>
+%%% @copyright (C) 2011, Rickard Green
+%%% @doc
+%%% Generation of the ethread atomic API
+%%% @end
+%%% Created : 17 Jan 2011 by Rickard Green <[email protected]>
+%%%-------------------------------------------------------------------
+
+-define(H_FILE, "erts/include/internal/ethr_atomics.h").
+-define(C_FILE, "erts/lib_src/common/ethr_atomics.c").
+
+%% These order constraints are important:
+%% - 'cmpxchg' needs to appear before 'read'
+%% - 'xchg' needs to apper before 'set'
+%% - 'set' needs to apper before 'init'
+%% - 'add_read' needs to apper before 'add', 'inc_read', and 'dec_read'
+%% - 'inc_read' needs to apper before and 'inc'
+%% - 'dec_read' needs to apper before and 'dec'
+-define(ATOMIC_OPS, [cmpxchg, xchg, set, init, add_read,
+ read, inc_read, dec_read, add, inc,
+ dec, read_band, read_bor]).
+
+-define(DW_ATOMIC_OPS, [cmpxchg, set, read, init]).
+-define(DW_FUNC_MACRO, "ETHR_DW_ATOMIC_FUNC__").
+-define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__").
+
+%% Barrier versions we implement
+-define(BARRIERS, [none, rb, wb, acqb, relb, mb]).
+
+-define(ATOMIC_SIZES, ["dword", "word", "32"]).
+
+-define(HAVE_NATIVE_ATOMIC, "ETHR_HAVE_ETHR_NATIVE_ATOMIC").
+
+-define(SU_DW_SINT_FIELD, "dw_sint").
+-define(DW_SINT_FIELD, "sint").
+
+%% Fallback
+-define(ETHR_ATMC_FLLBK_ADDR_BITS, "10").
+-define(ETHR_ATMC_FLLBK_ADDR_SHIFT, "6").
+
+-record(atomic_context, {dw,
+ amc_fallback,
+ ret_type,
+ ret_var,
+ arg1,
+ arg2,
+ arg3,
+ have_native_atomic_ops,
+ atomic,
+ atomic_t,
+ addr_aint_t,
+ aint_t,
+ naint_t,
+ 'NATMC',
+ 'ATMC',
+ unusual_val}).
+
+atomic_context("dword") ->
+ #atomic_context{dw = true,
+ amc_fallback = true,
+ ret_type = "int",
+ ret_var = "res",
+ arg1 = "var",
+ arg2 = "val",
+ arg3 = "old_val",
+ have_native_atomic_ops = "ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS",
+ atomic = "ethr_dw_atomic",
+ atomic_t = "ethr_dw_atomic_t",
+ addr_aint_t = "ethr_sint_t",
+ aint_t = "ethr_dw_sint_t",
+ naint_t = "ETHR_SU_DW_NAINT_T__",
+ 'NATMC' = "DW_NATMC",
+ 'ATMC' = "DW_ATMC",
+ unusual_val = "ETHR_UNUSUAL_SINT_VAL__"};
+atomic_context(Size) ->
+ {SizeSuffix, HaveSize, AMC} = case Size of
+ "word" -> {"", "WORD_SZ", true};
+ _ -> {Size, Size++"BIT", false}
+ end,
+ AintT = ["ethr_sint", SizeSuffix, "_t"],
+ #atomic_context{dw = false,
+ amc_fallback = AMC,
+ ret_type = AintT,
+ ret_var = "res",
+ arg1 = "var",
+ arg2 = "val",
+ arg3 = "old_val",
+ have_native_atomic_ops = ["ETHR_HAVE_", HaveSize, "_NATIVE_ATOMIC_OPS"],
+ atomic = ["ethr_atomic", SizeSuffix],
+ atomic_t = ["ethr_atomic", SizeSuffix, "_t"],
+ addr_aint_t = AintT,
+ aint_t = AintT,
+ naint_t = ["ETHR_NAINT", SizeSuffix, "_T__"],
+ 'NATMC' = ["NATMC", SizeSuffix],
+ 'ATMC' = ["ATMC", SizeSuffix],
+ unusual_val = ["ETHR_UNUSUAL_SINT", SizeSuffix, "_VAL__"]}.
+
+-record(op_context, {ret, var, val1, val2}).
+
+-define(POTENTIAL_NBITS, ["64", "32"]).
+
+is_return_op(#atomic_context{dw = false}, add) -> false;
+is_return_op(#atomic_context{dw = false}, inc) -> false;
+is_return_op(#atomic_context{dw = false}, dec) -> false;
+is_return_op(#atomic_context{dw = true}, read) -> false;
+is_return_op(_AC, init) -> false;
+is_return_op(_AC, set) -> false;
+is_return_op(_AC, _OP) -> true.
+
+native(add_read) -> add_return;
+native(inc_read) -> inc_return;
+native(dec_read) -> dec_return;
+native(read_band) -> and_retold;
+native(read_bor) -> or_retold;
+native(Op) -> Op.
+
+op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set ->
+ [Var, " = ", Val1];
+op(read, #op_context{ret = Ret, var = Var}) ->
+ [Ret, " = ", Var];
+op(add_read, OpC) ->
+ [op(add, OpC), "; ", op(read, OpC)];
+op(add, #op_context{var = Var, val1 = Val1}) ->
+ [Var, " += ", Val1];
+op(inc, #op_context{var = Var}) ->
+ ["++(", Var, ")"];
+op(dec, #op_context{var = Var}) ->
+ ["--(", Var, ")"];
+op(inc_read, #op_context{ret = Ret, var = Var}) ->
+ [Ret, " = ++(", Var, ")"];
+op(dec_read, #op_context{ret = Ret, var = Var}) ->
+ [Ret, " = --(", Var, ")"];
+op(read_band, #op_context{var = Var, val1 = Val1} = OpC) ->
+ [op(read, OpC), "; ", Var, " &= ", Val1];
+op(read_bor, #op_context{var = Var, val1 = Val1} = OpC) ->
+ [op(read, OpC), "; ", Var, " |= ", Val1];
+op(xchg, OpC) ->
+ [op(read, OpC), "; ", op(set, OpC)];
+op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) ->
+ [Ret, " = (", Var, " == ", Val2, " ? (", Var, " = ", Val1, ", ", Val2, ") : ", Var, ")"].
+
+dw_op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set ->
+ [Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]"];
+dw_op(read, #op_context{var = Var, val1 = Val1}) ->
+ [Val1, "[0] = ", Var, "[0]; ", Val1, "[1] = ", Var, "[1]"];
+dw_op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) ->
+ ["
+ {
+ ", Ret, " = (", Var, "[0] == ", Val2, "[0] && ", Var, "[1] == ", Val2, "[1]);
+ if (", Ret, ") {
+ ", Var, "[0] = ", Val1, "[0];
+ ", Var, "[1] = ", Val1, "[1];
+ }
+ else {
+ ", Val2, "[0] = ", Var, "[0];
+ ", Val2, "[1] = ", Var, "[1];
+ }
+ }"].
+
+op_head_tail(init) -> {undef, undef};
+op_head_tail(set) -> {store, store};
+op_head_tail(read) -> {load, load};
+op_head_tail(_) -> {load, undef}.
+
+op_barrier_ext(none) -> "";
+op_barrier_ext(Barrier) -> [$_, a2l(Barrier)].
+
+op_call(addr, _DW, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) ->
+ [Ret, " ", Func, "(", Arg1, ");"];
+op_call(Op, false, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == read;
+ Op == inc_read;
+ Op == inc_return;
+ Op == dec_read;
+ Op == dec_return ->
+ [Ret, " ", Func, "(", Arg1, ");"];
+op_call(Op, false, _Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == inc;
+ Op == dec ->
+ [Func, "(", Arg1, ");"];
+op_call(Op, false, Ret, Func, Arg1, Arg2, _Arg3, TypeCast) when Op == add_return;
+ Op == add_read;
+ Op == read_band;
+ Op == and_retold;
+ Op == read_bor;
+ Op == or_retold;
+ Op == xchg ->
+ [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"];
+op_call(cmpxchg, _DW, Ret, Func, Arg1, Arg2, Arg3, TypeCast) ->
+ [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ",", TypeCast, " ", Arg3, ");"];
+op_call(_Op, _DW, _Ret, Func, Arg1, Arg2, _Arg3, TypeCast) ->
+ [Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]. % set, init, add (!= dw), read (== dw)
+
+native_op_call(#atomic_context{dw = DW,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3,
+ aint_t = AintT,
+ 'NATMC' = NATMC,
+ naint_t = NAintT},
+ Op, B, TypeCasts) ->
+ op_call(Op,
+ DW,
+ [RetVar, " =",
+ case TypeCasts of
+ true -> [" (", AintT, ")"];
+ false -> ""
+ end],
+ ["ETHR_", NATMC, "_FUNC__(", opstr(native(Op)), op_barrier_ext(B), ")"],
+ Arg1,
+ Arg2,
+ Arg3,
+ case TypeCasts of
+ true -> [" (", NAintT, ")"];
+ false -> ""
+ end).
+
+simple_fallback(#atomic_context{arg1 = Arg1,
+ arg2 = Arg2,
+ 'ATMC' = ATMC},
+ init, B) -> %% Also double word
+ [" ETHR_", ATMC, "_FUNC__(set", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ 'ATMC' = ATMC},
+ set, B) ->
+ [" (void) ETHR_", ATMC, "_FUNC__(xchg", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ 'ATMC' = ATMC},
+ add, B) ->
+ [" (void) ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ aint_t = AintT,
+ 'ATMC' = ATMC},
+ inc_read, B) ->
+ [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") 1);\n"];
+simple_fallback(#atomic_context{dw = false,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ aint_t = AintT,
+ 'ATMC' = ATMC},
+ dec_read, B) ->
+ [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") -1);\n"];
+simple_fallback(#atomic_context{dw = false,
+ arg1 = Arg1,
+ 'ATMC' = ATMC},
+ inc, B) ->
+ [" (void) ETHR_", ATMC, "_FUNC__(inc_read", op_barrier_ext(B), ")(", Arg1, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+ arg1 = Arg1,
+ 'ATMC' = ATMC},
+ dec, B) ->
+ [" (void) ETHR_", ATMC, "_FUNC__(dec_read", op_barrier_ext(B), ")(", Arg1, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+ unusual_val = UnusualVal,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ aint_t = AintT,
+ 'ATMC' = ATMC},
+ read, B) ->
+ [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", (", AintT, ") ", UnusualVal, ", (", AintT,") ", UnusualVal, ");\n"];
+simple_fallback(#atomic_context{dw = true,
+ unusual_val = UnusualVal,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ aint_t = AintT,
+ 'ATMC' = ATMC},
+ read, B) ->
+ [" ", AintT, " tmp;
+ tmp.", ?DW_SINT_FIELD, "[0] = ", UnusualVal, ";
+ tmp.", ?DW_SINT_FIELD, "[1] = ", UnusualVal, ";
+ ", Arg2, "->", ?DW_SINT_FIELD, "[0] = ", UnusualVal, ";
+ ", Arg2, "->", ?DW_SINT_FIELD, "[1] = ", UnusualVal, ";
+ (void) ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", &tmp, ", Arg2, ");
+"
+ ];
+simple_fallback(_AC, _Op, _B) ->
+ [].
+
+func_header(AC, prototype, MacroName, Op, B) ->
+ [func_header(AC, implementation, MacroName, Op, B), ";"];
+func_header(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _MacroName, Op, B) ->
+ do_func_header(AC, Op, "static ETHR_INLINE ",
+ ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"]);
+func_header(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B) ->
+ do_func_header(AC, Op, "", [Atomic, "_", opstr(Op), op_barrier_ext(B)]);
+func_header(AC, implementation, MacroName, Op, B) ->
+ do_func_header(AC, Op, "", [MacroName, "(", opstr(Op), op_barrier_ext(B), ")"]).
+
+
+do_func_header(#atomic_context{atomic_t = AtomicT,
+ addr_aint_t = AddrAintT,
+ arg1 = Arg1},
+ addr, Inline, Func) ->
+ [Inline, AddrAintT, " *", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1,
+ arg2 = Arg2},
+ Op, Inline, Func) when Op == init;
+ Op == set;
+ Op == add ->
+ [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"];
+do_func_header(#atomic_context{dw = false,
+ atomic_t = AtomicT,
+ arg1 = Arg1},
+ Op, Inline, Func) when Op == inc;
+ Op == dec ->
+ [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1},
+ Op, Inline, Func) when Op == read;
+ Op == inc_read;
+ Op == dec_read ->
+ [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1,
+ arg2 = Arg2},
+ Op, Inline, Func) when Op == add_read;
+ Op == read_band;
+ Op == read_bor;
+ Op == xchg ->
+ [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"];
+do_func_header(#atomic_context{dw = false,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3},
+ cmpxchg, Inline, Func) ->
+ [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ", ", AintT, " ", Arg3, ")"];
+do_func_header(#atomic_context{dw = true,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1,
+ arg2 = Arg2},
+ Op, Inline, Func) when Op == init;
+ Op == set;
+ Op == read ->
+ [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ")"];
+do_func_header(#atomic_context{dw = true,
+ atomic_t = AtomicT,
+ aint_t = AintT,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3},
+ cmpxchg, Inline, Func) ->
+ [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"].
+
+
+xbarriers(_Op, none, _NB) ->
+ {"", ""};
+
+xbarriers(_Op, acqb, NB) when NB == acqb; NB == mb ->
+ {"", ""};
+xbarriers(Op, acqb, NB) ->
+ case {op_head_tail(Op), NB} of
+ {{_, load}, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore);"};
+ {{_, load}, _} -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"};
+ {{_, store}, _} -> {"", "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"};
+ {_, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"};
+ _ -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"}
+ end;
+
+xbarriers(_Op, relb, NB) when NB == relb; NB == mb ->
+ {"", ""};
+xbarriers(Op, relb, NB) ->
+ case {op_head_tail(Op), NB} of
+ {{store, _}, wb} -> {"ETHR_MEMBAR(ETHR_LoadStore);", ""};
+ {{store, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);", ""};
+ {{load, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);", ""};
+ {_, wb} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);", ""};
+ _ -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", ""}
+ end;
+
+xbarriers(_Op, wb, NB) when NB == wb; NB == mb ->
+ {"", ""};
+xbarriers(_Op, wb, _NB) ->
+ {"ETHR_MEMBAR(ETHR_StoreStore);", ""};
+
+xbarriers(_Op, rb, NB) when NB == rb; NB == mb ->
+ {"", ""};
+xbarriers(_Op, rb, _NB) ->
+ {"", "ETHR_MEMBAR(ETHR_LoadLoad);"};
+
+xbarriers(_Op, mb, mb) ->
+ {"", ""};
+xbarriers(Op, mb, NB) ->
+ MB = "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);",
+ {Head, Tail} = op_head_tail(Op),
+ PreOp = case {Head, NB} of
+ {_, relb} -> "";
+ {store, wb} -> "ETHR_MEMBAR(ETHR_LoadStore);";
+ {store, _} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);";
+ {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);";
+ {_, wb} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);";
+ _ -> MB
+ end,
+ PostOp = case {Tail, NB} of
+ {_, acqb} -> "";
+ {load, rb} -> "ETHR_MEMBAR(ETHR_LoadStore);";
+ {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);";
+ {store, _} -> "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);";
+ {_, rb} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);";
+ _ -> MB
+ end,
+ {PreOp, PostOp}.
+
+try_barrier_order_first(none) ->
+ [none, rb, wb, acqb, relb];
+try_barrier_order_first(acqb) ->
+ [acqb, rb, none, mb];
+try_barrier_order_first(relb) ->
+ [relb, wb, none, mb];
+try_barrier_order_first(rb) ->
+ [rb, none, mb];
+try_barrier_order_first(wb) ->
+ [wb, none, mb];
+try_barrier_order_first(mb) ->
+ [mb, relb, acqb, wb, rb, none].
+
+try_barrier_order(B) ->
+ First = try_barrier_order_first(B),
+ First ++ (?BARRIERS -- First).
+
+native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) ->
+ NOpStr = opstr(native(Op)),
+ CapNOpStr = to_upper(NOpStr),
+ NBExt = op_barrier_ext(NB),
+ CapNBExt = to_upper(NBExt),
+ {PreB, PostB} = xbarriers(Op, B, NB),
+ [If, " defined(ETHR_HAVE_", NATMC, "_", CapNOpStr, CapNBExt, ")\n",
+ ExtraDecl,
+ case PreB of
+ "" -> "";
+ _ -> [" ", PreB, "\n"]
+ end,
+ " ", native_op_call(AC, Op, NB, TypeCasts), "\n",
+ case PostB of
+ "" -> "";
+ _ -> [" ", PostB, "\n"]
+ end].
+
+dw_native_barrier_op(#atomic_context{arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, If, ExtraDecl, Op, B, NB) ->
+ native_barrier_op(AC#atomic_context{arg1 = ["&", Arg1, "->native"],
+ arg2 = [Arg2, "->", ?DW_SINT_FIELD],
+ arg3 = [Arg3, "->", ?DW_SINT_FIELD]},
+ If, ExtraDecl, Op, B, NB, false).
+
+su_dw_native_barrier_op(#atomic_context{dw = true,
+ naint_t = NAintT,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3,
+ 'NATMC' = NATMC} = AC, If, cmpxchg, B, NB) ->
+ SU = ["->", ?SU_DW_SINT_FIELD],
+ TmpVar = "act",
+ SUArg1 = ["&", Arg1, "->native"],
+ SUArg2 = [Arg2, SU],
+ SUArg3 = [Arg3, SU],
+ ExtraDecl = [" ", NAintT, " ", TmpVar, ";\n"],
+ [native_barrier_op(AC#atomic_context{dw = false,
+ ret_var = TmpVar,
+ arg1 = SUArg1,
+ arg2 = SUArg2,
+ arg3 = SUArg3,
+ 'NATMC' = ["SU_", NATMC]},
+ If, ExtraDecl, cmpxchg, B, NB, false),
+ " ", RetVar, " = (", TmpVar, " == ", SUArg3, ");
+ ", SUArg3, " = ", TmpVar, ";
+"
+ ];
+su_dw_native_barrier_op(#atomic_context{dw = true,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ 'NATMC' = NATMC} = AC, If, Op, B, NB) ->
+ SUArg1 = ["&", Arg1, "->native"],
+ SUArg2 = [Arg2, "->", ?SU_DW_SINT_FIELD],
+ native_barrier_op(AC#atomic_context{dw = false,
+ ret_var = SUArg2,
+ arg1 = SUArg1,
+ arg2 = SUArg2,
+ arg3 = not_used,
+ 'NATMC' = ["SU_", NATMC]}, If, "", Op, B, NB, false).
+
+cmpxchg_fallback_define(#atomic_context{dw = false, aint_t = AintT} = AC) ->
+ do_cmpxchg_fallback_define(AC, true, AintT);
+cmpxchg_fallback_define(#atomic_context{dw = true,
+ 'NATMC' = NATMC,
+ naint_t = NAintT} = AC) ->
+ ["\n\n#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)\n",
+ do_cmpxchg_fallback_define(AC, false, not_used),
+ "\n\n#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)\n",
+ do_cmpxchg_fallback_define(AC#atomic_context{'NATMC' = ["SU_", NATMC],
+ naint_t = NAintT},
+ true,
+ NAintT),
+ "
+
+#else
+# error \"?!?\"
+#endif
+"].
+
+do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC,
+ aint_t = AintT,
+ naint_t = NAintT},
+ SU, SUType) ->
+
+ ReadFunc = fun (IF) ->
+ fun (B) ->
+ BExt = op_barrier_ext(B),
+ CapBExt = to_upper(BExt),
+ [IF, " defined(ETHR_HAVE_", NATMC, "_READ", CapBExt, ")",
+ case SU of
+ true -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\
+ ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR)
+"
+ ];
+ false -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+ ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_", NATMC, "_READ", CapBExt, ")
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+ VAL.", ?SU_DW_SINT_FIELD, " = ETHR_SU_", NATMC, "_FUNC__(read", BExt, ")(VAR)
+"
+ ]
+ end]
+ end
+ end,
+ NotDefCMPXCHG = fun (B) ->
+ CapBExt = to_upper(op_barrier_ext(B)),
+ ["!defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapBExt, ")"]
+ end,
+ NoneTryBarrierOrder = try_barrier_order(none),
+ %% First a sanity check
+ ["
+#if (", NotDefCMPXCHG(hd(?BARRIERS)) ,
+ lists:map(fun (B) ->
+ [" \\
+ && ", NotDefCMPXCHG(B)]
+ end,
+ tl(?BARRIERS)), ")
+# error \"No native cmpxchg() op available\"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+",
+
+ %% Read op to use with cmpxchg fallback
+ (ReadFunc("#if"))(hd(NoneTryBarrierOrder)),
+ lists:map(ReadFunc("#elif"), tl(NoneTryBarrierOrder)),
+"#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */",
+ case SU of
+ true -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\
+ ((", NAintT, ") 0)"];
+ false -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+do { \\
+ VAL.", ?DW_SINT_FIELD, "[0] = (ethr_sint_t) 0; \\
+ VAL.", ?DW_SINT_FIELD, "[1] = (ethr_sint_t) 0; \\
+} while (0)"]
+ end, "
+#endif
+",
+
+ %% The fallback
+ "
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \\
+do { \\",
+ case SU of
+ true -> ["
+ ", SUType, " AVAL; \\
+ ", NAintT, " new__, act__, exp__; \\
+ act__ = ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR); \\
+ do { \\
+ exp__ = act__; \\
+ AVAL = (", SUType, ") act__; \\
+ { OPS; } \\
+ new__ = (", NAintT, ") AVAL; \\
+ act__ = CMPXCHG(VAR, new__, exp__); \\
+ } while (__builtin_expect(act__ != exp__, 0)); \\"];
+ false -> ["
+ int res__; \\
+ ", AintT, " AVAL, exp_act__; \\
+ ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \\
+ do { \\
+ AVAL.", ?DW_SINT_FIELD, "[0] = exp_act__.", ?DW_SINT_FIELD, "[0]; \\
+ AVAL.", ?DW_SINT_FIELD, "[1] = exp_act__.", ?DW_SINT_FIELD, "[1]; \\
+ { OPS; } \\
+ res__ = CMPXCHG(VAR, AVAL.", ?DW_SINT_FIELD, ", exp_act__.", ?DW_SINT_FIELD, "); \\
+ } while (__builtin_expect(res__ == 0, 0)); \\"]
+ end, "
+} while (0)
+"
+ ].
+
+cmpxchg_fallbacks(#atomic_context{}, _SUDW, cmpxchg, _B) ->
+ ""; %% No need for a fallback
+cmpxchg_fallbacks(#atomic_context{dw = DW,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3,
+ 'NATMC' = NATMC},
+ SUDW, Op, B) ->
+ Operation = case DW of
+ false ->
+ op(Op, #op_context{ret = RetVar,
+ var = "aval",
+ val1 = Arg2,
+ val2 = Arg3});
+ true ->
+ case SUDW of
+ true ->
+ op(Op, #op_context{ret = [Arg2, "->", ?SU_DW_SINT_FIELD],
+ var = "aval",
+ val1 = [Arg2, "->", ?SU_DW_SINT_FIELD]});
+ false ->
+ dw_op(Op, #op_context{ret = RetVar,
+ var = ["aval.", ?DW_SINT_FIELD],
+ val1 = [Arg2, "->", ?DW_SINT_FIELD]})
+ end
+ end,
+ [lists:map(fun (NB) ->
+ NativeVar = case DW of
+ true -> ["&", Arg1, "->native"];
+ false -> Arg1
+ end,
+ NBExt = op_barrier_ext(NB),
+ CapNBExt = to_upper(NBExt),
+ {PreB, PostB} = xbarriers(cmpxchg, B, NB),
+ ["#elif defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapNBExt, ")\n",
+ case PreB of
+ "" -> "";
+ _ -> [" ", PreB, "\n"]
+ end,
+ " ETHR_", NATMC, "_CMPXCHG_FALLBACK__(ETHR_", NATMC, "_FUNC__(cmpxchg", NBExt, "), ", NativeVar, ", aval, ", Operation, ");\n",
+ case PostB of
+ "" -> "";
+ _ -> [" ", PostB, "\n"]
+ end]
+ end,
+ try_barrier_order(B))].
+
+translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) ->
+ ["
+#if !defined(ETHR_", NATMC, "_BITS__)
+# error \"Missing native atomic implementation\"",
+ lists:map(fun (NBits) ->
+ {HaveInPrefix,
+ HaveOutPrefix,
+ HaveInPrefixExtra,
+ HaveOutPrefixExtra,
+ NativeTypeCheck} = case NBits of
+ "dw" ->
+ {"ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC",
+ ["ETHR_HAVE_", NATMC],
+ "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC",
+ ["ETHR_HAVE_SU_", NATMC],
+ "\n#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)"};
+ _ ->
+ {[?HAVE_NATIVE_ATOMIC, NBits],
+ case DW of
+ true -> ["ETHR_HAVE_SU_", NATMC];
+ false -> ["ETHR_HAVE_", NATMC]
+ end,
+ false,
+ ["ETHR_HAVE_", NATMC],
+ ["\n#elif ETHR_", NATMC, "_BITS__ == ", NBits]}
+ end,
+ [NativeTypeCheck,
+ lists:map(fun (Op) ->
+ NOpStr = opstr(native(Op)),
+ CapNOpStr = to_upper(NOpStr),
+ lists:map(fun (B) ->
+ NBExt = op_barrier_ext(B),
+ CapNBExt = to_upper(NBExt),
+ HaveOutDef = [HaveOutPrefix, "_", CapNOpStr, CapNBExt],
+ HaveOutDefExtra = [HaveOutPrefixExtra, "_", CapNOpStr, CapNBExt],
+ [case DW of
+ true ->
+ ["\n# undef ", HaveOutDefExtra];
+ false ->
+ ""
+ end, "
+# undef ", HaveOutDef,"
+# ifdef ", HaveInPrefix, "_", CapNOpStr, CapNBExt, "
+# define ", HaveOutDef, " 1
+# endif",
+ case HaveInPrefixExtra of
+ false -> "";
+ _ -> ["
+# ifdef ", HaveInPrefixExtra, "_", CapNOpStr, CapNBExt, "
+# define ", HaveOutDefExtra, " 1
+# endif"
+ ]
+ end]
+ end,
+ ?BARRIERS)
+ end,
+ case DW of
+ true -> ?DW_ATOMIC_OPS;
+ false -> ?ATOMIC_OPS
+ end)]
+ end,
+ case DW of
+ true -> ["dw", "64"];
+ false -> ?POTENTIAL_NBITS
+ end),
+ "
+#else
+# error \"Invalid native atomic size\"
+#endif
+"].
+
+
+
+make_prototypes(#atomic_context{dw = DW, 'ATMC' = ATMC} = AC) ->
+ MkProt = fun (MacroName) ->
+ %% addr() is special
+ [func_header(AC, prototype, MacroName, addr, none), "\n",
+ lists:map(fun (Op) ->
+ lists:map(fun (B) ->
+ [func_header(AC, prototype, MacroName, Op, B), "\n"]
+ end,
+ ?BARRIERS)
+ end,
+ case DW of
+ true -> ?DW_ATOMIC_OPS;
+ false -> ?ATOMIC_OPS
+ end)]
+ end,
+ ["
+#ifdef ETHR_NEED_", ATMC, "_PROTOTYPES__
+",
+ MkProt(false),
+ case DW of
+ true -> ["#if defined(", ?DW_RTCHK_MACRO, ")\n",
+ MkProt(?DW_FUNC_MACRO),
+ "#endif\n"];
+ false -> ""
+ end,
+ "#endif /* ETHR_NEED_", ATMC, "_PROTOTYPES__ */\n"].
+
+rtchk_fallback_call(Return, #atomic_context{dw = DW,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3},
+ Op, B) ->
+ op_call(Op, DW, case Return of
+ true -> "return";
+ false -> [RetVar, " ="]
+ end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, "").
+
+make_implementations(#atomic_context{dw = DW,
+ ret_type = RetType,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ addr_aint_t = AddrAintT,
+ atomic = Atomic,
+ have_native_atomic_ops = HaveNativeAtomicOps,
+ 'ATMC' = ATMC,
+ 'NATMC' = NATMC} = AC) ->
+ NativeVar = case DW of
+ true -> ["(&", Arg1, "->native)"];
+ false -> Arg1
+ end,
+ RtchkBegin = ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+ if (", ?DW_RTCHK_MACRO, ") {
+#endif
+"],
+ RtchkEnd = fun (Return, Operation, Barrier) ->
+ ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+ } else { ", rtchk_fallback_call(Return, AC, Operation, Barrier), " }
+#endif\n"
+ ]
+ end,
+ ["
+#if (defined(", HaveNativeAtomicOps, ") \\
+ && (defined(ETHR_", ATMC, "_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+",
+ translate_have_defs(AC),
+ cmpxchg_fallback_define(AC),
+ %% addr() is special
+ "
+
+
+/* --- addr() --- */
+
+", func_header(AC, inline_implementation, false, addr, none), "
+{", case DW of
+ true -> RtchkBegin;
+ false -> ""
+ end, "
+ return (", AddrAintT, " *) ETHR_", NATMC, "_ADDR_FUNC__(", NativeVar, ");
+",case DW of
+ true -> RtchkEnd(true, addr, none);
+ false -> ""
+ end, "
+}
+",
+ lists:map(fun (Op) ->
+ OpStr = opstr(Op),
+ ["
+
+/* --- ", OpStr, "() --- */
+
+",
+ lists:map(fun (B) ->
+ TryBarriers = try_barrier_order(B),
+ ["
+", func_header(AC, inline_implementation, false, Op, B), "
+{
+",
+ case is_return_op(AC, Op) of
+ true ->
+ [" ", RetType, " ", RetVar, ";\n"];
+ _ -> ""
+ end,
+ case DW of
+ true ->
+ [RtchkBegin,
+ "\n",
+ su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
+ lists:map(fun (NB) ->
+ su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
+ end,
+ tl(TryBarriers)),
+ lists:map(fun (NB) ->
+ dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
+ end,
+ TryBarriers),
+ case simple_fallback(AC, Op, B) of
+ "" ->
+ %% No simple fallback available;
+ %% use cmpxchg() fallbacks...
+ [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
+ cmpxchg_fallbacks(AC, false, Op, B),
+ "#else
+#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
+#endif
+"
+ ];
+ SimpleFallback ->
+ ["#else\n", SimpleFallback, "#endif\n"]
+ end,
+ RtchkEnd(false, Op, B), "\n"];
+ false ->
+ [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
+ lists:map(fun (NB) ->
+ native_barrier_op(AC, "#elif", "", Op, B, NB, true)
+ end,
+ tl(TryBarriers)),
+ case simple_fallback(AC, Op, B) of
+ "" ->
+ %% No simple fallback available;
+ %% use cmpxchg() fallbacks...
+ [cmpxchg_fallbacks(AC, false, Op, B),
+ "#else
+#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
+#endif
+"
+ ];
+ SimpleFallback ->
+ ["#else\n", SimpleFallback, "#endif\n"]
+ end]
+ end,
+ case is_return_op(AC, Op) of
+ true ->
+ [" return ", RetVar, ";\n"];
+ false ->
+ ""
+ end,
+ "}\n"]
+ end,
+ ?BARRIERS)]
+ end,
+ case DW of
+ true -> ?DW_ATOMIC_OPS;
+ false -> ?ATOMIC_OPS
+ end),
+ "
+#endif /* ETHR_", ATMC, "_INLINE__ */
+"
+ ].
+
+atomic_implementation_comment(AtomicSize) ->
+ CSz = case AtomicSize of
+ "dword" -> "Double word size";
+ "word" -> "Word size";
+ _ -> AtomicSize ++ "-bit"
+ end,
+ ["
+
+/* ---------- ", CSz, " atomic implementation ---------- */
+
+"
+ ].
+
+write_h_file(FileName) ->
+ {ok, FD} = file:open(FileName, [write, latin1]),
+ ok = file:write(FD, comments()),
+ ok = file:write(FD, "
+#ifndef ETHR_ATOMICS_H__
+#define ETHR_ATOMICS_H__
+"
+ ),
+ ok = file:write(FD, h_top()),
+ ok = lists:foreach(fun (AtomicSize) ->
+ AC = atomic_context(AtomicSize),
+ ok = file:write(FD,
+ [atomic_implementation_comment(AtomicSize),
+ make_prototypes(AC),
+ make_implementations(AC)])
+ end,
+ ?ATOMIC_SIZES),
+ ok = file:write(FD, "
+#endif /* ETHR_ATOMICS_H__ */
+"
+ ),
+ ok = file:close(FD).
+
+
+make_native_impl_op(#atomic_context{dw = DW,
+ atomic = Atomic,
+ have_native_atomic_ops = HaveNativeAtomicOps,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3}, Op, B) ->
+ ["#if defined(", HaveNativeAtomicOps, ")",
+ case DW of
+ true -> [" && !defined(", ?DW_RTCHK_MACRO, ")"];
+ false -> ""
+ end,
+ "\n",
+ " ", op_call(Op, DW, [RetVar, " = "], [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""),
+ "\n"].
+
+amc_op_dw_arg(#atomic_context{dw = false}) ->
+ "0";
+amc_op_dw_arg(#atomic_context{dw = true}) ->
+ "1".
+
+amc_op_arg_prefix(#atomic_context{dw = false}) ->
+ "&";
+amc_op_arg_prefix(#atomic_context{dw = true}) ->
+ "".
+
+amc_sint_arg(#atomic_context{dw = DW, arg2 = Arg}, arg2) ->
+ amc_sint_arg(DW, Arg);
+amc_sint_arg(#atomic_context{dw = DW, arg3 = Arg}, arg3) ->
+ amc_sint_arg(DW, Arg);
+amc_sint_arg(#atomic_context{dw = DW, ret_var = Arg}, ret_var) ->
+ amc_sint_arg(DW, Arg);
+amc_sint_arg(true, Arg) ->
+ [Arg, "->" ?DW_SINT_FIELD];
+amc_sint_arg(false, Arg) ->
+ ["&", Arg].
+
+amc_op_call(#atomic_context{arg1 = Arg1} = AC, init) ->
+ [" amc_init(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"];
+amc_op_call(#atomic_context{arg1 = Arg1} = AC, set) ->
+ [" amc_set(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"];
+amc_op_call(#atomic_context{dw = false, arg1 = Arg1} = AC, read) ->
+ [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, ret_var), ");\n"];
+amc_op_call(#atomic_context{dw = true, arg1 = Arg1} = AC, read) ->
+ [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"];
+amc_op_call(#atomic_context{dw = false, arg1 = Arg1, arg3 = Arg3, ret_var = RetVar} = AC, cmpxchg) ->
+ [" ", RetVar, " = ", Arg3, ";
+ (void) amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, ret_var), ");\n"];
+amc_op_call(#atomic_context{dw = true, arg1 = Arg1, ret_var = RetVar} = AC, cmpxchg) ->
+ [" ", RetVar, " = amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, arg3), ");\n"];
+amc_op_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar}, Op) ->
+ OpCtxt = #op_context{ret = RetVar, var = [Arg1,"->sint"], val1 = Arg2, val2 = Arg3},
+ OpStr = case DW of
+ true -> dw_op(Op, OpCtxt);
+ false -> op(Op, OpCtxt)
+ end,
+ [" ETHR_AMC_MODIFICATION_OPS__(&", Arg1, "->amc, ", OpStr, ");\n"].
+
+make_amc_fallback_op(#atomic_context{amc_fallback = false}, _Op, _B) ->
+ "";
+make_amc_fallback_op(#atomic_context{amc_fallback = true} = AC, Op, B) ->
+ NB = case Op of
+ read -> rb;
+ _ -> none
+ end,
+ {PreB, PostB} = xbarriers(Op, B, NB),
+ ["#elif defined(ETHR_AMC_FALLBACK__)\n",
+ case PreB of
+ "" -> "";
+ _ -> [" ", PreB, "\n"]
+ end,
+ amc_op_call(AC, Op),
+ case PostB of
+ "" -> "";
+ _ -> [" ", PostB, "\n"]
+ end].
+
+make_locked_fallback_op(#atomic_context{dw = DW,
+ ret_var = RetVar,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3}, Op, B) ->
+ OpStr = case DW of
+ true ->
+ dw_op(Op, #op_context{ret = RetVar,
+ var = [Arg1, "->" ?DW_SINT_FIELD],
+ val1 = [Arg2, "->" ?DW_SINT_FIELD],
+ val2 = [Arg3, "->" ?DW_SINT_FIELD]});
+ false ->
+ op(Op, #op_context{ret = RetVar,
+ var = ["*", Arg1],
+ val1 = Arg2,
+ val2 = Arg3})
+ end,
+ {PreB, PostB} = xbarriers(Op, B, none),
+ ["#else\n",
+ case PreB of
+ "" -> "";
+ _ -> [" ", PreB, "\n"]
+ end,
+ [" ETHR_ATOMIC_OP_FALLBACK_IMPL__(", Arg1, ", ", OpStr, ");\n"],
+ case PostB of
+ "" -> "";
+ _ -> [" ", PostB, "\n"]
+ end,
+ "#endif\n"].
+
+make_symbol_to_fallback_impl(#atomic_context{dw = true,
+ atomic = Atomic,
+ arg1 = Arg1,
+ arg2 = Arg2,
+ arg3 = Arg3} = AC,
+ Op, B) ->
+ ["
+#ifdef ", ?DW_RTCHK_MACRO, "
+", func_header(AC, implementation, false, Op, B), "
+{",
+ case Op of
+ init -> "";
+ _ -> ["\n ETHR_ASSERT(!ethr_not_inited__);"]
+ end, "
+ ETHR_ASSERT(", Arg1, ");
+ ", op_call(Op, true, "return", [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), "
+}
+#endif
+"
+ ];
+make_symbol_to_fallback_impl(_, _, _) ->
+ "".
+
+make_symbol_implementations(#atomic_context{dw = DW,
+ amc_fallback = AMC,
+ ret_type = RetType,
+ addr_aint_t = AddrAintT,
+ ret_var = RetVar,
+ arg1 = Arg1} = AC) ->
+ FallbackVar = case DW of
+ true -> ["(&", Arg1, "->fallback)"];
+ false -> Arg1
+ end,
+ ["
+",
+ case DW of
+ true -> ["
+/*
+ * Double word atomics need runtime test.
+ */
+
+int ethr_have_native_dw_atomic(void)
+{
+ return ethr_have_native_dw_atomic__();
+}
+ "];
+ false -> ""
+ end, "
+
+/* --- addr() --- */
+
+", func_header(AC, implementation,
+ case DW of
+ true -> ?DW_FUNC_MACRO;
+ false -> false
+ end, addr, none), "
+{
+ ", AddrAintT, " *", RetVar, ";
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(", Arg1, ");
+", make_native_impl_op(AC, addr, none),
+ case AMC of
+ true -> ["#elif defined(ETHR_AMC_FALLBACK__)
+ ", RetVar ," = (", AddrAintT, " *) (", FallbackVar, ")->sint;"];
+ false -> ""
+ end, "
+#else
+ ", RetVar, " = (", AddrAintT, " *) ", FallbackVar, ";
+#endif
+ return ", RetVar, ";
+}
+",
+ make_symbol_to_fallback_impl(AC, addr, none),
+ lists:map(fun (Op) ->
+ ["
+
+/* -- ", opstr(Op), "() -- */
+
+",
+ lists:map(fun (B) ->
+ ["\n",
+ func_header(AC, implementation,
+ case DW of
+ true -> ?DW_FUNC_MACRO;
+ false -> false
+ end, Op, B),
+ "\n{\n",
+ case is_return_op(AC, Op) of
+ true -> [" ", RetType, " ", RetVar, ";\n"];
+ false -> ""
+ end,
+ case Op of
+ init -> "";
+ _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"]
+ end,
+ [" ETHR_ASSERT(", Arg1, ");\n"],
+ make_native_impl_op(AC, Op, B),
+ make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
+ make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
+ case is_return_op(AC, Op) of
+ true -> [" return ", RetVar, ";"
+ ];
+ false ->
+ ""
+ end,
+ "\n}\n",
+ make_symbol_to_fallback_impl(AC, Op, B)]
+ end,
+ ?BARRIERS)]
+ end,
+ case DW of
+ true -> ?DW_ATOMIC_OPS;
+ false -> ?ATOMIC_OPS
+ end)].
+
+make_info_functions() ->
+ ["
+
+
+/* --------- Info functions --------- */
+
+#if defined(", ?DW_RTCHK_MACRO, ")
+char *zero_ops[] = {NULL};
+#endif
+",
+ [lists:map(fun (NBits) ->
+ {DW, Bits} = case NBits of
+ "su_dw" -> {"su_dw_", ""};
+ "dw" -> {"dw_", ""};
+ _ -> {"", NBits}
+ end,
+ ["
+
+static char *native_", DW, "atomic", Bits, "_ops[] = {",
+ lists:map(fun (Op) ->
+ NOpStr = opstr(native(Op)),
+ CapNOpStr = to_upper(NOpStr),
+ lists:map(fun (B) ->
+ HaveNative = case NBits of
+ "dw" ->
+ "ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC";
+ "su_dw" ->
+ "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC";
+ _ ->
+ [?HAVE_NATIVE_ATOMIC, NBits]
+ end,
+ NBExt = op_barrier_ext(B),
+ CapNBExt = to_upper(NBExt),
+ ["
+#ifdef ", HaveNative, "_", CapNOpStr, CapNBExt, "
+ \"", NOpStr, NBExt, "\",
+#endif"
+ ]
+ end,
+ ?BARRIERS)
+ end,
+ case NBits of
+ "dw" -> ?DW_ATOMIC_OPS;
+ "su_dw" -> ?DW_ATOMIC_OPS;
+ _ -> ?ATOMIC_OPS
+ end), "
+ NULL
+};
+
+char **
+ethr_native_", DW, "atomic", Bits, "_ops(void)
+{
+",
+ case DW of
+ "" -> "";
+ _ -> ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+ if (!", ?DW_RTCHK_MACRO, ")
+ return &zero_ops[0];
+#endif"
+ ]
+ end, "
+ return &native_", DW, "atomic", Bits, "_ops[0];
+}
+"
+ ]
+ end, ["su_dw", "dw" | ?POTENTIAL_NBITS])]].
+
+write_c_file(FileName) ->
+ {ok, FD} = file:open(FileName, [write, latin1]),
+ ok = file:write(FD, comments()),
+ ok = file:write(FD, c_top()),
+ lists:foreach(fun (AtomicSize) ->
+ ok = file:write(FD,
+ [atomic_implementation_comment(AtomicSize),
+ make_symbol_implementations(atomic_context(AtomicSize))])
+ end,
+ ?ATOMIC_SIZES),
+ ok = file:write(FD, make_info_functions()).
+
+
+main([]) ->
+ case os:getenv("ERL_TOP") of
+ false ->
+ io:format("$ERL_TOP not set!~n", []),
+ halt(1);
+ ErlTop ->
+ HFile = filename:join(ErlTop, ?H_FILE),
+ WHFile = fun () ->
+ write_h_file(HFile)
+ end,
+ CFile = filename:join(ErlTop, ?C_FILE),
+ WCFile = fun () ->
+ write_c_file(CFile)
+ end,
+ case erlang:system_info(schedulers_online) of
+ 1 ->
+ WHFile(),
+ WCFile();
+ _ ->
+ {HPid, HMon} = spawn_monitor(WHFile),
+ {CPid, CMon} = spawn_monitor(WCFile),
+ receive
+ {'DOWN', HMon, process, HPid, HReason} ->
+ normal = HReason
+ end,
+ receive
+ {'DOWN', CMon, process, CPid, CReason} ->
+ normal = CReason
+ end
+ end,
+ io:format("Wrote: ~s~n", [HFile]),
+ io:format("Wrote: ~s~n", [CFile]),
+ init:stop()
+ end.
+
+a2l(A) ->
+ atom_to_list(A).
+
+opstr(A) ->
+ a2l(A).
+
+to_upper([]) ->
+ [];
+to_upper([C|Cs]) when is_list(C) ->
+ [to_upper(C)|to_upper(Cs)];
+to_upper([C|Cs]) when is_integer(C), 97 =< C, C =< 122 ->
+ [C-32|to_upper(Cs)];
+to_upper([C|Cs]) ->
+ [C|to_upper(Cs)].
+
+
+comments() ->
+ Years = case erlang:date() of
+ {2011, _, _} -> "2011";
+ {Y, _, _} -> "2011-"++integer_to_list(Y)
+ end,
+ ["/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * \$ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB ", Years, ". All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the \"License\"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an \"AS IS\"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomics API
+ * Author: Rickard Green
+ */
+
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+",
+ lists:map(fun (Op) ->
+ [" * - ", opstr(Op), "\n"]
+ end,
+ ?ATOMIC_OPS),
+ " *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+",
+ lists:map(fun (Op) ->
+ [" * - ", opstr(Op), "\n"]
+ end,
+ ?DW_ATOMIC_OPS),
+ " *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * memory barrier semantics:
+",
+ lists:map(fun (none) ->
+ "";
+ (rb) ->
+ [" * - rb (read barrier)\n"];
+ (wb) ->
+ [" * - wb (write barrier)\n"];
+ (acqb) ->
+ [" * - acqb (acquire barrier)\n"];
+ (relb) ->
+ [" * - relb (release barrier)\n"];
+ (mb) ->
+ [" * - mb (full memory barrier)\n"];
+ (B) ->
+ [" * - ", a2l(B), "\n"]
+ end,
+ ?BARRIERS),
+ " *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ * ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ * ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ * ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ * and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ * either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ * (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ * A macro informing about the presence of the native
+ * implementation:
+ *
+ * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ * A macro naming (a string constant) the implementation:
+ *
+ * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ * Each implemented native atomic function has to
+ * be accompanied by a defined macro on the following
+ * form informing about its presence:
+ *
+ * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ * A (sparc-v9 style) membar macro:
+ *
+ * - ETHR_MEMBAR(B)
+ *
+ * Which takes a combination of the following macros
+ * or:ed (using |) together:
+ *
+ * - ETHR_LoadLoad
+ * - ETHR_LoadStore
+ * - ETHR_StoreLoad
+ * - ETHR_StoreStore
+ *
+ */
+"
+ ].
+
+h_top() ->
+ ["
+#undef ETHR_AMC_FALLBACK__
+#undef ETHR_AMC_NO_ATMCS__
+#undef ETHR_AMC_ATMC_T__
+#undef ETHR_AMC_ATMC_FUNC__
+
+/* -- 32-bit atomics -- */
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+#undef ETHR_NATMC32_ADDR_FUNC__
+#undef ETHR_NATMC32_BITS__
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+# define ETHR_NEED_NATMC32_ADDR
+# define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic32_t;
+# define ETHR_NAINT32_T__ ethr_sint32_t
+# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_NATMC32_BITS__ 32
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# define ETHR_NEED_NATMC64_ADDR
+#ifdef ETHR_BIGENDIAN
+# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\
+ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+# define ETHR_NATMC32_ADDR_FUNC__(VAR) \\
+ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic32_t;
+# define ETHR_NAINT32_T__ ethr_sint64_t
+# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC32_BITS__ 64
+#else
+/*
+ * No native atomics usable for 32-bits atomics :(
+ * Use fallback...
+ */
+typedef ethr_sint32_t ethr_atomic32_t;
+#endif
+
+#undef ETHR_ATMC32_INLINE__
+#ifdef ETHR_NATMC32_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC32_INLINE__
+# endif
+# define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_ATMC32_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
+# define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC32_FUNC__
+#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+
+/* -- Word size atomics -- */
+
+#undef ETHR_NEED_NATMC32_ADDR
+#undef ETHR_NEED_NATMC64_ADDR
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_NATMC_ADDR_FUNC__
+#undef ETHR_NATMC_BITS__
+#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
+typedef ethr_native_atomic64_t ethr_atomic_t;
+# define ETHR_NAINT_T__ ethr_sint64_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC_BITS__ 64
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC32_ADDR
+# endif
+# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic_t;
+# define ETHR_NAINT_T__ ethr_sint32_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+# define ETHR_NATMC_BITS__ 32
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+#ifdef ETHR_BIGENDIAN
+# define ETHR_NATMC_ADDR_FUNC__(VAR) \\
+ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+# define ETHR_NATMC_ADDR_FUNC__(VAR) \\
+ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic_t;
+# define ETHR_NATMC_T__ ethr_native_atomic64_t
+# define ETHR_NAINT_T__ ethr_sint64_t
+# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_NATMC_BITS__ 64
+#else
+/*
+ * No native atomics usable for pointer size atomics :(
+ * Use fallback...
+ */
+
+# if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 2
+# define ETHR_AMC_SINT_T__ ethr_sint32_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+typedef struct {
+ ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+typedef struct {
+ ethr_amc_t amc;
+ ethr_sint_t sint;
+} ethr_atomic_t;
+# else /* locked fallback */
+typedef ethr_sint_t ethr_atomic_t;
+# endif
+#endif
+
+#undef ETHR_ATMC_INLINE__
+#ifdef ETHR_NATMC_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC_INLINE__
+# endif
+# define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC_FUNC_NAME_
+# define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC_FUNC__
+#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+
+/* -- Double word atomics -- */
+
+#undef ETHR_SU_DW_NAINT_T__
+#undef ETHR_SU_DW_NATMC_FUNC__
+#undef ETHR_SU_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_FUNC__
+#undef ETHR_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_BITS__
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+# define ETHR_NEED_DW_NATMC_ADDR
+# define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
+# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
+# define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
+# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
+# if ETHR_SIZEOF_PTR == 8
+# define ETHR_DW_NATMC_BITS__ 128
+# elif ETHR_SIZEOF_PTR == 4
+# define ETHR_DW_NATMC_BITS__ 64
+# else
+# error \"Word size not supported\"
+# endif
+# ifdef ETHR_NATIVE_SU_DW_SINT_T
+# define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
+# endif
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+# ifndef ETHR_NEED_NATMC64_ADDR
+# define ETHR_NEED_NATMC64_ADDR
+# endif
+# define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \\
+ ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
+# define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
+# define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
+# define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+# define ETHR_DW_NATMC_BITS__ 64
+#endif
+
+#if defined(", ?DW_RTCHK_MACRO, ")
+#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X ## _fallback__
+#else
+#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X
+#endif
+
+#if !defined(ETHR_DW_NATMC_BITS__) || defined(", ?DW_RTCHK_MACRO, ")
+# define ETHR_NEED_DW_FALLBACK__
+#endif
+
+#if defined(ETHR_NEED_DW_FALLBACK__)
+/*
+ * No native atomics usable for double word atomics :(
+ * Use fallback...
+ */
+
+# ifndef ETHR_AMC_FALLBACK__
+# if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 1
+# define ETHR_AMC_SINT_T__ ethr_sint_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+# elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+# define ETHR_AMC_FALLBACK__
+# define ETHR_AMC_NO_ATMCS__ 2
+# define ETHR_AMC_SINT_T__ ethr_sint32_t
+# define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+# define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+# endif
+# ifdef ETHR_AMC_FALLBACK__
+typedef struct {
+ ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+# endif
+# endif
+
+typedef struct {
+#ifdef ETHR_AMC_FALLBACK__
+ ethr_amc_t amc;
+#endif
+ ethr_sint_t sint[2];
+} ethr_dw_atomic_fallback_t;
+
+#endif
+
+typedef union {
+#ifdef ETHR_NATIVE_DW_ATOMIC_T__
+ ETHR_NATIVE_DW_ATOMIC_T__ native;
+#endif
+#ifdef ETHR_NEED_DW_FALLBACK__
+ ethr_dw_atomic_fallback_t fallback;
+#endif
+ ethr_sint_t sint[2];
+} ethr_dw_atomic_t;
+
+typedef union {
+#ifdef ETHR_SU_DW_NAINT_T__
+ ETHR_SU_DW_NAINT_T__ ", ?SU_DW_SINT_FIELD, ";
+#endif
+ ethr_sint_t ", ?DW_SINT_FIELD, "[2];
+} ethr_dw_sint_t;
+
+#ifdef ETHR_BIGENDIAN
+# define ETHR_DW_SINT_LOW_WORD 1
+# define ETHR_DW_SINT_HIGH_WORD 0
+#else
+# define ETHR_DW_SINT_LOW_WORD 0
+# define ETHR_DW_SINT_HIGH_WORD 1
+#endif
+
+#undef ETHR_DW_ATMC_INLINE__
+#ifdef ETHR_DW_NATMC_BITS__
+# ifdef ETHR_TRY_INLINE_FUNCS
+# define ETHR_ATMC32_INLINE__
+# endif
+# define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_DW_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
+# define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_DW_ATMC_FUNC__
+#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)
+
+#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
+int ethr_have_native_dw_atomic(void);
+#endif
+#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+static ETHR_INLINE int
+ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
+{
+#if defined(", ?DW_RTCHK_MACRO, ")
+ return ", ?DW_RTCHK_MACRO, ";
+#elif defined(ETHR_DW_NATMC_BITS__)
+ return 1;
+#else
+ return 0;
+#endif
+}
+#endif
+
+/* -- Misc -- */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+/*
+ * Unusual values are used by read() fallbacks implemented via cmpxchg().
+ * We want to use an unusual value in hope that it is more efficient
+ * not to match the value in memory.
+ *
+ * - Negative integer values are probably more unusual.
+ * - Very large absolute integer values are probably more unusual.
+ * - Odd pointers are probably more unusual (only char pointers can be odd).
+ */
+# define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
+# if ETHR_SIZEOF_PTR == 4
+# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
+# elif ETHR_SIZEOF_PTR == 8
+# define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
+# else
+# error \"Word size not supported\"
+# endif
+# if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
+# error \"No ethr_native_dw_atomic_addr() available\"
+# endif
+# if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
+# error \"No ethr_native_atomic32_addr() available\"
+# endif
+# if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
+# error \"No ethr_native_atomic64_addr() available\"
+# endif
+#endif
+
+#if defined(__GNUC__)
+# ifndef ETHR_COMPILER_BARRIER
+# define ETHR_COMPILER_BARRIER __asm__ __volatile__(\"\" : : : \"memory\")
+# endif
+#elif defined(ETHR_WIN32_THREADS)
+# ifndef ETHR_COMPILER_BARRIER
+# include <intrin.h>
+# pragma intrinsic(_ReadWriteBarrier)
+# define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+# endif
+#endif
+
+void ethr_compiler_barrier_fallback(void);
+#ifndef ETHR_COMPILER_BARRIER
+# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
+#endif
+
+int ethr_init_atomics(void);
+
+/* info */
+char **ethr_native_atomic32_ops(void);
+char **ethr_native_atomic64_ops(void);
+char **ethr_native_dw_atomic_ops(void);
+char **ethr_native_su_dw_atomic_ops(void);
+
+#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
+/*
+ * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
+ * i.e. when no native atomic implementation exist and only our lock
+ * based atomic fallback is used, a noop is sufficient.
+ */
+# undef ETHR_MEMORY_BARRIER
+# undef ETHR_WRITE_MEMORY_BARRIER
+# undef ETHR_READ_MEMORY_BARRIER
+# undef ETHR_READ_DEPEND_MEMORY_BARRIER
+# undef ETHR_MEMBAR
+# define ETHR_MEMBAR(B) do { } while (0)
+#endif
+
+#ifndef ETHR_MEMBAR
+# error \"No ETHR_MEMBAR defined\"
+#endif
+
+#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
+#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
+#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
+#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
+# undef ETHR_ORDERED_READ_DEPEND
+#else
+# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+# define ETHR_ORDERED_READ_DEPEND
+#endif
+"].
+
+c_top() ->
+ ["
+
+#ifdef HAVE_CONFIG_H
+#include \"config.h\"
+#endif
+
+#define ETHR_TRY_INLINE_FUNCS
+#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
+#define ETHR_ATOMIC_IMPL__
+
+#include \"ethread.h\"
+#include \"ethr_internal.h\"
+
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\
+ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+/*
+ * Spinlock based fallback for atomics used in absence of a native
+ * implementation.
+ */
+
+#define ETHR_ATMC_FLLBK_ADDR_BITS ", ?ETHR_ATMC_FLLBK_ADDR_BITS, "
+#define ETHR_ATMC_FLLBK_ADDR_SHIFT ", ?ETHR_ATMC_FLLBK_ADDR_SHIFT, "
+
+typedef struct {
+ union {
+ ethr_spinlock_t lck;
+ char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
+ } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \\
+(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \\
+ & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \\
+do { \\
+ ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \\
+ ethr_spin_lock(slp__); \\
+ { EXPS; } \\
+ ethr_spin_unlock(slp__); \\
+} while (0)
+
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#endif
+
+", make_amc_fallback(), "
+
+int
+ethr_init_atomics(void)
+{
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\
+ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+ int i;
+ for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
+ int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+ if (res != 0)
+ return res;
+ }
+#endif
+ return 0;
+}
+"].
+
+make_amc_fallback() ->
+ ["
+#if defined(ETHR_AMC_FALLBACK__)
+
+/*
+ * Fallback for large sized (word and/or double word size) atomics using
+ * an \"Atomic Modification Counter\" based on smaller sized native atomics.
+ *
+ * We use a 63-bit modification counter and a one bit exclusive flag.
+ * If 32-bit native atomics are used, we need two 32-bit native atomics.
+ * The exclusive flag is the least significant bit, or if multiple atomics
+ * are used, the least significant bit of the least significant atomic.
+ *
+ * When using the AMC fallback the following is true:
+ * - Reads of the same atomic variable can be done in parallel.
+ * - Uncontended reads doesn't cause any cache line invalidations,
+ * since no modifications are done.
+ * - Assuming that the AMC atomic(s) and the integer(s) containing the
+ * value of the implemented atomic resides in the same cache line,
+ * modifications will only cause invalidations of one cache line.
+ *
+ * When using the spinlock based fallback none of the above is true,
+ * however, the spinlock based fallback consumes less memory.
+ */
+
+# if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2
+# error \"Not supported\"
+# endif
+# define ETHR_AMC_MAX_TRY_READ__ 10
+# ifdef ETHR_DEBUG
+# define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \\
+do { \\
+ ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \\
+ ETHR_ASSERT(act == (S) + 1); \\
+ ETHR_ASSERT(act & 1); \\
+} while (0)
+# else
+# define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
+# endif
+
+static ETHR_INLINE void
+amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ avar[0] = val[0];
+ if (dw)
+ avar[1] = val[1];
+#if ETHR_AMC_NO_ATMCS__ == 2
+ ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
+#endif
+ ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
+}
+
+static ETHR_INLINE ETHR_AMC_SINT_T__
+amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
+{
+ ETHR_AMC_SINT_T__ state0 = prev_state0;
+ /* Set exclusive flag. */
+ while (1) {
+ ETHR_AMC_SINT_T__ act_state0, new_state0;
+ while (state0 & 1) { /* Wait until exclusive bit has been cleared */
+ ETHR_SPIN_BODY;
+ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+ }
+ /* Try to set exclusive bit */
+ new_state0 = state0 + 1;
+ act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
+ new_state0,
+ state0);
+ if (state0 == act_state0)
+ return state0; /* old state0 */
+ state0 = act_state0;
+ }
+}
+
+static ETHR_INLINE void
+amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+ ETHR_AMC_SINT_T__ state0 = old_state0;
+
+ /* Increment modification counter and reset exclusive flag. */
+
+ ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+ state0 += 2;
+
+ ETHR_ASSERT((state0 & 1) == 0);
+
+#if ETHR_AMC_NO_ATMCS__ == 2
+ if (state0 == 0) {
+ /*
+ * state0 wrapped, so we need to increment state1. There is no need
+ * for atomic inc op, since this is always done while having exclusive
+ * flag.
+ */
+ ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
+ state1++;
+ ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
+ }
+#endif
+ ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
+}
+
+static ETHR_INLINE void
+amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+ ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
+ /*
+ * Reset exclusive flag, but leave modification counter unchanged,
+ * i.e., restore state to what it was before setting exclusive
+ * flag.
+ */
+ ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
+}
+
+static ETHR_INLINE void
+amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+
+ state0 = amc_set_excl(amc, state0);
+
+ avar[0] = val[0];
+ if (dw)
+ avar[1] = val[1];
+
+ amc_inc_mc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+ ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
+{
+ /* *state0p should contain last read value if aborting */
+ ETHR_AMC_SINT_T__ old_state0;
+#if ETHR_AMC_NO_ATMCS__ == 2
+ ETHR_AMC_SINT_T__ state1;
+ int abrt;
+#endif
+
+ *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+ if ((*state0p) & 1)
+ return 0; /* exclusive flag set; abort */
+#if ETHR_AMC_NO_ATMCS__ == 2
+ state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
+#else
+ ETHR_COMPILER_BARRIER;
+#endif
+
+ val[0] = avar[0];
+ if (dw)
+ val[1] = avar[1];
+
+ ETHR_READ_MEMORY_BARRIER;
+
+ /*
+ * Abort if state has changed (i.e, either the exclusive
+ * flag is set, or modification counter changed).
+ */
+ old_state0 = *state0p;
+#if ETHR_AMC_NO_ATMCS__ == 2
+ *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+ abrt = (old_state0 != *state0p);
+ abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
+ return abrt == 0;
+#else
+ *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+ return old_state0 == *state0p;
+#endif
+}
+
+static ETHR_INLINE void
+amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+ ETHR_AMC_SINT_T__ state0;
+ int i;
+
+#if ETHR_AMC_MAX_TRY_READ__ == 0
+ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+#else
+ for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
+ if (amc_try_read(amc, dw, avar, val, &state0))
+ return; /* read success */
+ ETHR_SPIN_BODY;
+ }
+#endif
+
+ state0 = amc_set_excl(amc, state0);
+
+ val[0] = avar[0];
+ if (dw)
+ val[1] = avar[1];
+
+ amc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+ ethr_sint_t *new, ethr_sint_t *xchg)
+{
+ ethr_sint_t val[2];
+ ETHR_AMC_SINT_T__ state0;
+
+ if (amc_try_read(amc, dw, avar, val, &state0)) {
+ if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
+ xchg[0] = val[0];
+ if (dw)
+ xchg[1] = val[1];
+ return 0; /* failed */
+ }
+ /* Operation will succeed if not interrupted */
+ }
+
+ state0 = amc_set_excl(amc, state0);
+
+ if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
+ xchg[0] = avar[0];
+ if (dw)
+ xchg[1] = avar[1];
+
+ ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+ amc_unset_excl(amc, state0);
+ return 0; /* failed */
+ }
+
+ avar[0] = new[0];
+ if (dw)
+ avar[1] = new[1];
+
+ amc_inc_mc_unset_excl(amc, state0);
+ return 1;
+}
+
+
+#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \\
+do { \\
+ ETHR_AMC_SINT_T__ state0__; \\
+ state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \\
+ state0__ = amc_set_excl((AMC), state0__); \\
+ { OPS; } \\
+ amc_inc_mc_unset_excl((AMC), state0__); \\
+} while (0)
+
+#endif /* amc fallback */
+"].
+
diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c
index 68f093f49c..bc2f635c26 100644
--- a/erts/lib_src/win/ethr_event.c
+++ b/erts/lib_src/win/ethr_event.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -28,13 +28,10 @@
/* --- Windows implementation of thread events ------------------------------ */
-#pragma intrinsic(_InterlockedExchangeAdd)
-#pragma intrinsic(_InterlockedCompareExchange)
-
int
ethr_event_init(ethr_event *e)
{
- e->state = ETHR_EVENT_OFF__;
+ ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__);
e->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
if (e->handle == INVALID_HANDLE_VALUE)
return ethr_win_get_errno__();
@@ -63,7 +60,6 @@ ethr_event_reset(ethr_event *e)
static ETHR_INLINE int
wait(ethr_event *e, int spincount)
{
- LONG state;
DWORD code;
int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -73,13 +69,9 @@ wait(ethr_event *e, int spincount)
sc = spincount;
while (1) {
- long on;
+ ethr_sint32_t state;
while (1) {
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
- state = e->state;
-#else
- state = _InterlockedExchangeAdd(&e->state, (LONG) 0);
-#endif
+ state = ethr_atomic32_read(&e->state);
if (state == ETHR_EVENT_ON__)
return 0;
if (sc == 0)
@@ -95,9 +87,9 @@ wait(ethr_event *e, int spincount)
}
if (state != ETHR_EVENT_OFF_WAITER__) {
- state = _InterlockedCompareExchange(&e->state,
- ETHR_EVENT_OFF_WAITER__,
- ETHR_EVENT_OFF__);
+ state = ethr_atomic32_cmpxchg(&e->state,
+ ETHR_EVENT_OFF_WAITER__,
+ ETHR_EVENT_OFF__);
if (state == ETHR_EVENT_ON__)
return 0;
ETHR_ASSERT(state == ETHR_EVENT_OFF__);
diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c
index 789a360b11..3abda6de4c 100644
--- a/erts/lib_src/win/ethread.c
+++ b/erts/lib_src/win/ethread.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -35,6 +35,7 @@
#include <winerror.h>
#include <stdio.h>
#include <limits.h>
+#include <intrin.h>
#define ETHR_INLINE_FUNC_NAME_(X) X ## __
#define ETHREAD_IMPL__
@@ -158,6 +159,25 @@ ethr_abort__(void)
#endif
}
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+#pragma intrinsic(__cpuid)
+
+void
+ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx)
+{
+ int CPUInfo[4];
+
+ __cpuid(CPUInfo, *eax);
+
+ *eax = CPUInfo[0];
+ *ebx = CPUInfo[1];
+ *ecx = CPUInfo[2];
+ *edx = CPUInfo[3];
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
/*
* ----------------------------------------------------------------------------
* Exported functions
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index cd44e8232c..f704135ce8 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
Binary files differ
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 5deb69edab..ff3633a3cc 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -35,14 +35,13 @@
-export([delay_trap/2]).
-export([set_cookie/2, get_cookie/0]).
-export([nodes/0]).
--export([concat_binary/1]).
+
-export([list_to_integer/2,integer_to_list/2]).
-export([flush_monitor_message/2]).
-export([set_cpu_topology/1, format_cpu_topology/1]).
-export([await_proc_exit/3]).
-deprecated([hash/2]).
--deprecated([concat_binary/1]).
% Get rid of autoimports of spawn to avoid clashes with ourselves.
-compile({no_auto_import,[spawn/1]}).
@@ -535,11 +534,6 @@ set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) ->
get_cookie() ->
auth:get_cookie().
--spec concat_binary(ListOfBinaries) -> binary() when
- ListOfBinaries :: iolist().
-concat_binary(List) ->
- list_to_binary(List).
-
-spec integer_to_list(Integer, Base) -> string() when
Integer :: integer(),
Base :: 2..36.
diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl
index 71d8c1c679..4206bebfe7 100644
--- a/erts/test/ethread_SUITE.erl
+++ b/erts/test/ethread_SUITE.erl
@@ -47,14 +47,29 @@
spinlock/1,
rwspinlock/1,
rwmutex/1,
- atomic/1]).
+ atomic/1,
+ dw_atomic_massage/1]).
-include_lib("test_server/include/test_server.hrl").
-tests() ->
- [create_join_thread, equal_tids, mutex, try_lock_mutex,
- cond_wait, broadcast, detached_thread,
- max_threads, tsd, spinlock, rwspinlock, rwmutex, atomic].
+tests() ->
+ [create_join_thread,
+ equal_tids,
+ mutex,
+ try_lock_mutex,
+ cond_wait,
+ broadcast,
+ detached_thread,
+ max_threads,
+ tsd,
+ spinlock,
+ rwspinlock,
+ rwmutex,
+ atomic,
+ dw_atomic_massage].
+
+all(doc) -> [];
+all(suite) -> tests().
suite() -> [{ct_hooks,[ts_install_cth]}].
@@ -211,6 +226,13 @@ atomic(suite) ->
atomic(Config) ->
run_case(Config, "atomic", "").
+dw_atomic_massage(doc) ->
+ ["Massage double word atomics"];
+dw_atomic_massage(suite) ->
+ [];
+dw_atomic_massage(Config) ->
+ run_case(Config, "dw_atomic_massage", "").
+
%%
%%
%% Auxiliary functions
diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c
index 0b59ff5aa6..7e7e133d6c 100644
--- a/erts/test/ethread_SUITE_data/ethread_tests.c
+++ b/erts/test/ethread_SUITE_data/ethread_tests.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -1310,6 +1310,9 @@ rwmutex_test(void)
* Tests atomics.
*/
+#define AT_AINT32_MAX 0x7fffffff
+#define AT_AINT32_MIN 0x80000000
+
#define AT_THREADS 4
#define AT_ITER 10000
@@ -1320,12 +1323,428 @@ static ethr_atomic_t at_go;
static ethr_atomic_t at_done;
static ethr_atomic_t at_data;
+#define AT_TEST_INIT(T, A, B) \
+do { \
+ ethr_ ## A ## _init ## B(&A, 17); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_SET(T, A, B) \
+do { \
+ ethr_ ## A ## _set ## B(&A, 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_XCHG(T, A, B) \
+do { \
+ ethr_ ## A ## _set ## B(&A, 4711); \
+ ASSERT(ethr_ ## A ## _xchg ## B(&A, 17) == 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_CMPXCHG(T, A, B) \
+do { \
+ ethr_ ## A ## _set ## B(&A, 4711); \
+ ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 33) == 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+ ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 4711) == 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_ADD_READ(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MAX; \
+ var_ += 4711; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+ ASSERT(ethr_ ## A ## _add_read ## B(&A, 4711) == var_); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ var_ = AT_AINT32_MIN; \
+ var_ -= 4711; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+ ASSERT(ethr_ ## A ## _add_read ## B(&A, -4711) == var_); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 4711); \
+ ASSERT(ethr_ ## A ## _add_read ## B(&A, 10) == 4721); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4721); \
+} while (0)
+
+#define AT_TEST_ADD(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MAX; \
+ var_ += 4711; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+ ethr_ ## A ## _add ## B(&A, 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ var_ = AT_AINT32_MIN; \
+ var_ -= 4711; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+ ethr_ ## A ## _add ## B(&A, -4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 11); \
+ ethr_ ## A ## _add ## B(&A, 4700); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_INC_READ(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MAX; \
+ var_++; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+ ASSERT(ethr_ ## A ## _inc_read ## B(&A) == var_); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 4710); \
+ ASSERT(ethr_ ## A ## _inc_read ## B(&A) == 4711); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_DEC_READ(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MIN; \
+ var_--; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+ ASSERT(ethr_ ## A ## _dec_read ## B(&A) == var_); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 17); \
+ ASSERT(ethr_ ## A ## _dec_read ## B(&A) == 16); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \
+} while (0)
+
+
+#define AT_TEST_INC(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MAX; \
+ var_++; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+ ethr_ ## A ## _inc ## B(&A); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 4710); \
+ ethr_ ## A ## _inc ## B(&A); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_DEC(T, A, B) \
+do { \
+ T var_ = AT_AINT32_MIN; \
+ var_--; \
+ ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+ ethr_ ## A ## _dec ## B(&A); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+ ethr_ ## A ## _set ## B(&A, 17); \
+ ethr_ ## A ## _dec ## B(&A); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \
+} while (0)
+
+#define AT_TEST_READ_BAND(T, A, B) \
+do { \
+ ethr_ ## A ## _set ## B(&A, 0x13131313); \
+ ASSERT(ethr_ ## A ## _read_band ## B(&A, 0x31313131) == 0x13131313); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 0x11111111); \
+} while (0)
+
+#define AT_TEST_READ_BOR(T, A, B) \
+do { \
+ ethr_ ## A ## _set ## B(&A, 0x11111111); \
+ ASSERT(ethr_ ## A ## _read_bor ## B(&A, 0x23232323) == 0x11111111); \
+ ASSERT(ethr_ ## A ## _read ## B(&A) == 0x33333333); \
+} while (0)
+
+
+static void
+atomic_basic_test(void)
+{
+ /*
+ * Verify that each op does what it is expected
+ * to do for at least one input.
+ */
+ ethr_atomic32_t atomic32;
+ ethr_atomic_t atomic;
+
+ print_line("AT_AINT32_MAX=%d",AT_AINT32_MAX);
+ print_line("AT_AINT32_MIN=%d",AT_AINT32_MIN);
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, );
+ AT_TEST_SET(ethr_sint32_t, atomic32, );
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, );
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, );
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, );
+ AT_TEST_ADD(ethr_sint32_t, atomic32, );
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, );
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, );
+ AT_TEST_INC(ethr_sint32_t, atomic32, );
+ AT_TEST_DEC(ethr_sint32_t, atomic32, );
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, );
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, );
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_SET(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_ADD(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_INC(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_DEC(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _acqb);
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _acqb);
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_SET(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_ADD(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_INC(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_DEC(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _relb);
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _relb);
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_SET(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_ADD(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_INC(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_DEC(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _rb);
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _rb);
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_SET(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_ADD(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_INC(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_DEC(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _wb);
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _wb);
+
+ AT_TEST_INIT(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_SET(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_XCHG(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_ADD(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_INC_READ(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_INC(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_DEC(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _mb);
+ AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _mb);
+
+ AT_TEST_INIT(ethr_sint_t, atomic, );
+ AT_TEST_SET(ethr_sint_t, atomic, );
+ AT_TEST_XCHG(ethr_sint_t, atomic, );
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, );
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, );
+ AT_TEST_ADD(ethr_sint_t, atomic, );
+ AT_TEST_INC_READ(ethr_sint_t, atomic, );
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, );
+ AT_TEST_INC(ethr_sint_t, atomic, );
+ AT_TEST_DEC(ethr_sint_t, atomic, );
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, );
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, );
+
+ AT_TEST_INIT(ethr_sint_t, atomic, _acqb);
+ AT_TEST_SET(ethr_sint_t, atomic, _acqb);
+ AT_TEST_XCHG(ethr_sint_t, atomic, _acqb);
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, _acqb);
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, _acqb);
+ AT_TEST_ADD(ethr_sint_t, atomic, _acqb);
+ AT_TEST_INC_READ(ethr_sint_t, atomic, _acqb);
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, _acqb);
+ AT_TEST_INC(ethr_sint_t, atomic, _acqb);
+ AT_TEST_DEC(ethr_sint_t, atomic, _acqb);
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, _acqb);
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, _acqb);
+
+ AT_TEST_INIT(ethr_sint_t, atomic, _relb);
+ AT_TEST_SET(ethr_sint_t, atomic, _relb);
+ AT_TEST_XCHG(ethr_sint_t, atomic, _relb);
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, _relb);
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, _relb);
+ AT_TEST_ADD(ethr_sint_t, atomic, _relb);
+ AT_TEST_INC_READ(ethr_sint_t, atomic, _relb);
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, _relb);
+ AT_TEST_INC(ethr_sint_t, atomic, _relb);
+ AT_TEST_DEC(ethr_sint_t, atomic, _relb);
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, _relb);
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, _relb);
+
+ AT_TEST_INIT(ethr_sint_t, atomic, _rb);
+ AT_TEST_SET(ethr_sint_t, atomic, _rb);
+ AT_TEST_XCHG(ethr_sint_t, atomic, _rb);
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, _rb);
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, _rb);
+ AT_TEST_ADD(ethr_sint_t, atomic, _rb);
+ AT_TEST_INC_READ(ethr_sint_t, atomic, _rb);
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, _rb);
+ AT_TEST_INC(ethr_sint_t, atomic, _rb);
+ AT_TEST_DEC(ethr_sint_t, atomic, _rb);
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, _rb);
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, _rb);
+
+ AT_TEST_INIT(ethr_sint_t, atomic, _wb);
+ AT_TEST_SET(ethr_sint_t, atomic, _wb);
+ AT_TEST_XCHG(ethr_sint_t, atomic, _wb);
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, _wb);
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, _wb);
+ AT_TEST_ADD(ethr_sint_t, atomic, _wb);
+ AT_TEST_INC_READ(ethr_sint_t, atomic, _wb);
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, _wb);
+ AT_TEST_INC(ethr_sint_t, atomic, _wb);
+ AT_TEST_DEC(ethr_sint_t, atomic, _wb);
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, _wb);
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, _wb);
+
+ AT_TEST_INIT(ethr_sint_t, atomic, _mb);
+ AT_TEST_SET(ethr_sint_t, atomic, _mb);
+ AT_TEST_XCHG(ethr_sint_t, atomic, _mb);
+ AT_TEST_CMPXCHG(ethr_sint_t, atomic, _mb);
+ AT_TEST_ADD_READ(ethr_sint_t, atomic, _mb);
+ AT_TEST_ADD(ethr_sint_t, atomic, _mb);
+ AT_TEST_INC_READ(ethr_sint_t, atomic, _mb);
+ AT_TEST_DEC_READ(ethr_sint_t, atomic, _mb);
+ AT_TEST_INC(ethr_sint_t, atomic, _mb);
+ AT_TEST_DEC(ethr_sint_t, atomic, _mb);
+ AT_TEST_READ_BAND(ethr_sint_t, atomic, _mb);
+ AT_TEST_READ_BOR(ethr_sint_t, atomic, _mb);
+
+ /* Double word */
+ {
+ ethr_dw_atomic_t dw_atomic;
+ ethr_dw_sint_t dw0, dw1;
+ dw0.sint[0] = 4711;
+ dw0.sint[1] = 4712;
+
+ /* init */
+ ethr_dw_atomic_init(&dw_atomic, &dw0);
+ ethr_dw_atomic_read(&dw_atomic, &dw1);
+ ETHR_ASSERT(dw1.sint[0] == 4711);
+ ETHR_ASSERT(dw1.sint[1] == 4712);
+
+ /* set */
+ dw0.sint[0] = 42;
+ dw0.sint[1] = ~((ethr_sint_t) 0);
+ ethr_dw_atomic_set(&dw_atomic, &dw0);
+ ethr_dw_atomic_read(&dw_atomic, &dw1);
+ ASSERT(dw1.sint[0] == 42);
+ ASSERT(dw1.sint[1] == ~((ethr_sint_t) 0));
+
+ /* cmpxchg */
+ dw0.sint[0] = 17;
+ dw0.sint[1] = 18;
+ dw1.sint[0] = 19;
+ dw1.sint[1] = 20;
+ ASSERT(!ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0));
+ ethr_dw_atomic_read(&dw_atomic, &dw0);
+ ASSERT(dw0.sint[0] == 42);
+ ASSERT(dw0.sint[1] == ~((ethr_sint_t) 0));
+
+ ASSERT(ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0));
+
+ ethr_dw_atomic_read(&dw_atomic, &dw0);
+ ASSERT(dw0.sint[0] == 19);
+ ASSERT(dw0.sint[1] == 20);
+ }
+}
+
+
+#define AT_DW_MIN 12
+#define AT_DW_MAX 42
+#define AT_DW_THREADS (AT_DW_MAX - AT_DW_MIN + 1)
+
+#define AT_DW_LOOPS 200000
+#define AT_DW_R_LOOPS 10
+
+ethr_dw_atomic_t at_dw_atomic;
+
+void
+at_dw_valid(ethr_dw_sint_t *dw)
+{
+ int i;
+ char c;
+ char *cp;
+
+ ASSERT(dw->sint[0] == dw->sint[1]);
+
+ cp = (char *) &dw->sint[0];
+ c = cp[0];
+
+ ASSERT(AT_DW_MIN <= c && c <= AT_DW_MAX);
+
+ for (i = 0; i < sizeof(ethr_sint_t); i++)
+ ASSERT(c == cp[i]);
+}
+
+void *
+at_dw_thr(void *vval)
+{
+ int l, r;
+ ethr_sint_t val = (ethr_sint_t) vval;
+ ethr_dw_sint_t dw;
+ ethr_dw_sint_t my_dw;
+
+ my_dw.sint[0] = val;
+ my_dw.sint[1] = val;
+
+ ethr_dw_atomic_set(&at_dw_atomic, &my_dw);
+ for (l = 0; l < AT_DW_LOOPS; l++) {
+ for (r = 0; r < AT_DW_R_LOOPS; r++) {
+ ethr_dw_atomic_read(&at_dw_atomic, &dw);
+ at_dw_valid(&dw);
+ }
+ ethr_dw_atomic_set(&at_dw_atomic, &my_dw);
+ for (r = 0; r < AT_DW_R_LOOPS; r++) {
+ ethr_dw_atomic_read(&at_dw_atomic, &dw);
+ at_dw_valid(&dw);
+ }
+ dw.sint[0] = 0;
+ dw.sint[1] = 0;
+ while (1) {
+ if (ethr_dw_atomic_cmpxchg(&at_dw_atomic, &my_dw, &dw))
+ break;
+ }
+ }
+}
+
+static void
+dw_atomic_massage_test(void)
+{
+ int i, res;
+ ethr_tid tid[AT_DW_THREADS];
+ ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
+ ethr_dw_sint_t dw;
+
+ dw.sint[0] = dw.sint[1] = 0;
+
+ ethr_dw_atomic_init(&at_dw_atomic, &dw);
+
+ for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) {
+ ethr_sint_t val;
+ memset(&val, i, sizeof(ethr_sint_t));
+ res = ethr_thr_create(&tid[i-AT_DW_MIN], at_dw_thr, (void *) val, &thr_opts);
+ ASSERT(res == 0);
+ }
+ for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) {
+ res = ethr_thr_join(tid[i-AT_DW_MIN], NULL);
+ ASSERT(res == 0);
+ }
+}
+
void *
at_thread(void *unused)
{
int i;
long val, go;
-
val = ethr_atomic_inc_read(&at_ready);
ASSERT(val > 0);
@@ -1373,7 +1792,6 @@ at_thread(void *unused)
return NULL;
}
-
static void
atomic_test(void)
{
@@ -1382,6 +1800,8 @@ atomic_test(void)
ethr_tid tid[AT_THREADS];
ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
+ atomic_basic_test();
+
#if ETHR_SIZEOF_PTR > 4
at_rm_val = ((long) 1) << 57;
at_set_val = ((long) 1) << 60;
@@ -1493,6 +1913,8 @@ main(int argc, char *argv[])
rwmutex_test();
else if (strcmp(testcase, "atomic") == 0)
atomic_test();
+ else if (strcmp(testcase, "dw_atomic_massage") == 0)
+ dw_atomic_massage_test();
else
skip("Test case \"%s\" not implemented yet", testcase);
diff --git a/lib/asn1/c_src/Makefile b/lib/asn1/c_src/Makefile
index 9e9cb18524..60543df355 100644
--- a/lib/asn1/c_src/Makefile
+++ b/lib/asn1/c_src/Makefile
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 2002-2010. All Rights Reserved.
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -51,33 +51,26 @@ EI_LIBDIR = $(ERL_TOP)/lib/erl_interface/obj$(TYPEMARKER)/$(TARGET)
# ----------------------------------------------------
# FLAGS
# ----------------------------------------------------
-EI_INCLUDES = -I$(ERL_TOP)/lib/erl_interface/include
CFLAGS = $(DED_INCLUDES) $(EI_INCLUDES) $(DED_CFLAGS)
LDFLAGS += $(DED_LDFLAGS)
-LD_INCL_EI = -L$(EI_LIBDIR)
-
# ----------------------------------------------------
# Target Specs
# ----------------------------------------------------
-C_FILES = asn1_erl_driver.c
+NIF_OBJ_FILES = $(OBJDIR)/asn1_erl_nif.o
ifeq ($(TARGET),win32)
-LD_EI = -lei_md
-SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.dll
-OBJ_FILES = $(OBJDIR)/asn1_erl_drv.o
+NIF_SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_nif.dll
CLIB_FLAGS =
LN=cp
else
-LD_EI = -lei
-OBJ_FILES = $(OBJDIR)/asn1_erl_drv.o
ifeq ($(findstring vxworks,$(TARGET)),vxworks)
-SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.eld
+NIF_SHARED_OBJ_FILE = $(LIBDIR)/asn1_erl_nif.eld
CLIB_FLAGS =
else
-SHARED_OBJ_FILES = $(LIBDIR)/asn1_erl_drv.so
+NIF_SHARED_OBJ_FILE = $(LIBDIR)/asn1_erl_nif.so
CLIB_FLAGS = -lc
endif
LN= ln -s
@@ -87,7 +80,7 @@ endif
# Targets
# ----------------------------------------------------
-opt: $(OBJDIR) $(LIBDIR) $(SHARED_OBJ_FILES)
+opt: $(OBJDIR) $(LIBDIR) $(NIF_SHARED_OBJ_FILE)
debug: opt
@@ -103,11 +96,11 @@ docs:
# ----------------------------------------------------
-$(OBJ_FILES): $(C_FILES)
- $(CC) -c $(CFLAGS) -o $(OBJ_FILES) $(C_FILES)
+$(OBJDIR)/%.o: %.c
+ $(CC) -c $(CFLAGS) -O3 -o $@ $<
-$(SHARED_OBJ_FILES): $(OBJ_FILES)
- $(LD) $(LDFLAGS) $(LD_INCL_EI) -o $(SHARED_OBJ_FILES) $(OBJ_FILES) $(LD_EI) $(CLIB_FLAGS) $(LIBS)
+$(NIF_SHARED_OBJ_FILE): $(NIF_OBJ_FILES)
+ $(LD) $(LDFLAGS) -o $(NIF_SHARED_OBJ_FILE) $(NIF_OBJ_FILES) $(CLIB_FLAGS) $(LIBS)
$(LIBDIR):
-mkdir -p $(LIBDIR)
@@ -124,9 +117,9 @@ include $(ERL_TOP)/make/otp_release_targets.mk
release_spec: opt
$(INSTALL_DIR) $(RELSYSDIR)/priv/lib
- $(INSTALL_PROGRAM) $(SHARED_OBJ_FILES) $(RELSYSDIR)/priv/lib
+ $(INSTALL_PROGRAM) $(NIF_SHARED_OBJ_FILE) $(RELSYSDIR)/priv/lib
$(INSTALL_DIR) $(RELSYSDIR)/c_src
- $(INSTALL_DATA) $(C_FILES) $(RELSYSDIR)/c_src
+ $(INSTALL_DATA) *.c $(RELSYSDIR)/c_src
release_docs_spec:
diff --git a/lib/asn1/c_src/asn1_erl_driver.c b/lib/asn1/c_src/asn1_erl_driver.c
deleted file mode 100644
index 18d4157941..0000000000
--- a/lib/asn1/c_src/asn1_erl_driver.c
+++ /dev/null
@@ -1,1677 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
- *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * %CopyrightEnd%
- *
- */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include "erl_driver.h"
-#include "ei.h"
-
-
-/* #define ASN1_DEBUG 1 */
-
-#define ASN1_OK 0
-#define ASN1_ERROR -1
-#define ASN1_COMPL_ERROR 1
-#define ASN1_MEMORY_ERROR 0
-#define ASN1_DECODE_ERROR 2
-#define ASN1_TAG_ERROR -3
-#define ASN1_LEN_ERROR -4
-#define ASN1_INDEF_LEN_ERROR -5
-#define ASN1_VALUE_ERROR -6
-
-
-#define ASN1_CLASS 0xc0
-#define ASN1_FORM 0x20
-#define ASN1_CLASSFORM (ASN1_CLASS | ASN1_FORM)
-#define ASN1_TAG 0x1f
-#define ASN1_LONG_TAG 0x7f
-
-#define ASN1_INDEFINITE_LENGTH 0x80
-#define ASN1_SHORT_DEFINITE_LENGTH 0
-
-#define ASN1_PRIMITIVE 0
-#define ASN1_CONSTRUCTED 0x20
-
-#define ASN1_COMPLETE 1
-#define ASN1_BER_TLV_DECODE 2
-#define ASN1_BER_TLV_PARTIAL_DECODE 3
-
-#define ASN1_NOVALUE 0
-
-#define ASN1_SKIPPED 0
-#define ASN1_OPTIONAL 1
-#define ASN1_CHOOSEN 2
-
-
-#define CEIL(X,Y) ((X-1) / Y + 1)
-
-#define INVMASK(X,M) (X & (M ^ 0xff))
-#define MASK(X,M) (X & M)
-
-typedef struct {
- ErlDrvPort port;
- int buffer_size;
-} asn1_data;
-
-/* int min_alloc_bytes; */
-
-
-static ErlDrvData asn1_drv_start(ErlDrvPort, char *);
-
-static void asn1_drv_stop(ErlDrvData);
-
-int asn1_drv_control(ErlDrvData, unsigned int, char *, int, char **, int);
-
-int complete(ErlDrvBinary **,unsigned char *,unsigned char *, int);
-
-int insert_octets(int, unsigned char **, unsigned char **, int *);
-
-int insert_octets_except_unused(int, unsigned char **, unsigned char **,
- int *, int);
-
-int insert_octets_as_bits_exact_len(int, int, unsigned char **,
- unsigned char **, int *);
-
-int insert_octets_as_bits(int, unsigned char **, unsigned char **,int *);
-
-int pad_bits(int, unsigned char **, int *);
-
-int insert_least_sign_bits(int, unsigned char, unsigned char **, int *);
-
-int insert_most_sign_bits(int, unsigned char, unsigned char **, int *);
-
-int insert_bits_as_bits(int, int, unsigned char **, unsigned char **, int *);
-
-int insert_octets_unaligned(int, unsigned char **, unsigned char **, int);
-
-int realloc_decode_buf(ErlDrvBinary **,int);
-
-int realloc_memory(ErlDrvBinary **,int,unsigned char **,unsigned char **);
-
-int decode_begin(ErlDrvBinary **,unsigned char *, int, unsigned int *);
-
-int decode(ErlDrvBinary **,int *,unsigned char *,int *, int);
-
-int decode_tag(char *,int *,unsigned char *,int,int *);
-
-int decode_value(int *,unsigned char *,int *,ErlDrvBinary **,int ,int);
-
-
-/* declaration of functions used for partial decode of a BER encoded
- message */
-
-int decode_partial(ErlDrvBinary **,unsigned char *, int);
-
-int skip_tag(unsigned char *,int *,int);
-
-int skip_length_and_value(unsigned char *,int *,int);
-
-int get_tag(unsigned char *,int *,int);
-
-int get_length(unsigned char *,int *,int *,int);
-
-int get_value(char *,unsigned char *,int *,int);
-
-static ErlDrvEntry asn1_drv_entry = {
- NULL, /* init, always NULL for dynamic drivers */
- asn1_drv_start, /* start, called when port is opened */
- asn1_drv_stop, /* stop, called when port is closed */
- NULL, /* output, called when erlang has sent */
- NULL, /* ready_input, called when input descriptor ready */
- NULL, /* ready_output, called when output descriptor ready */
- "asn1_erl_drv", /* char *driver_name, the argument to open_port */
- NULL, /* finish, called when unloaded */
- NULL, /* void * that is not used (BC) */
- asn1_drv_control, /* control, port_control callback */
- NULL, /* timeout, called on timeouts */
- NULL, /* outputv, vector output interface */
-
- NULL, /* ready_async */
- NULL, /* flush */
- NULL, /* call */
- NULL, /* event */
- ERL_DRV_EXTENDED_MARKER,
- ERL_DRV_EXTENDED_MAJOR_VERSION,
- ERL_DRV_EXTENDED_MINOR_VERSION,
- ERL_DRV_FLAG_USE_PORT_LOCKING,
- NULL, /* handle2 */
- NULL /* process_exit */
-};
-
-
-
-DRIVER_INIT(asn1_erl_drv) /* must match name in driver_entry */
-{
- return &asn1_drv_entry;
-}
-
-static ErlDrvData asn1_drv_start(ErlDrvPort port, char *buff)
-{
- /* extern int min_alloc_bytes; */
- char *ptr;
- asn1_data* d;
-
- d = (asn1_data*)driver_alloc(sizeof(asn1_data));
- set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY);
- d->port = port;
-
- if ((ptr = getenv("ASN1_MIN_BUF_SIZE")) == NULL)
- d->buffer_size = 1024;
- else
- d->buffer_size = atoi(ptr);
- return (ErlDrvData)d;
-}
-
-
-static void asn1_drv_stop(ErlDrvData handle)
-{
- driver_free((char*)handle);
-}
-
-
-
-int asn1_drv_control(ErlDrvData handle,
- unsigned int command,
- char *buf,
- int buf_len,
- char **res_buf,
- int res_buf_len)
-{
- unsigned char *complete_buf;
- int complete_len, decode_len;
- ErlDrvBinary *drv_binary;
- ErlDrvBinary **drv_bin_ptr;
- asn1_data* a_data;
- int min_alloc_bytes;
- unsigned int err_pos = 0; /* in case of error, return last correct position */
- int ret_err; /* return value in case of error in TLV decode, i.e. length of list in res_buf */
-
- /* In case previous call to asn1_drv_control resulted in a change of
- return value from binary to integer list */
- a_data = (asn1_data *)handle;
- min_alloc_bytes = a_data->buffer_size;
- set_port_control_flags(a_data->port, PORT_CONTROL_FLAG_BINARY);
-
- if (command == ASN1_COMPLETE)
- {
- if (buf_len==0) {
- return 0; /* Avoid binary buffer overwrite (OTP-8451) */
- }
- /* Do the PER complete encode step */
- if ((drv_binary = driver_alloc_binary(buf_len))==NULL) {
- /* error handling */
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }
- complete_buf = (unsigned char*) drv_binary->orig_bytes;
- if ((complete_len = complete(&drv_binary,complete_buf,(unsigned char*) buf,buf_len)) == ASN1_ERROR)
- {
- /* error handling due to failure in complete */
- /* printf("error when running complete\n\r"); */
- driver_free_binary(drv_binary);
- set_port_control_flags(a_data->port, 0);
- **res_buf = '1';
- return ASN1_COMPL_ERROR;
- }
- /* printf("complete_len=%dbuf_len=%d,orig_size=%d\n\r",complete_len,buf_len,drv_binary->orig_size); */
- /* now the message is complete packed, return to Erlang */
- /* if (complete_len < buf_len) {*/
- if (complete_len < drv_binary->orig_size) {
- ErlDrvBinary *tmp;
- if ((tmp=driver_realloc_binary(drv_binary,complete_len)) == NULL){
- /*error handling due to memory allocation failure */
- driver_free_binary(drv_binary);
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }else
- drv_binary=tmp;
- }
- *res_buf = (char *)drv_binary;
- return complete_len;
- } else if (command == ASN1_BER_TLV_DECODE) { /* control == 2 */
- /* Do the tlv decode,
- return the resulting term encoded on the Erlang
- external format */
-/* printf("driver: buffer_len = %d, min_alloc_bytes = %d\r\n",buf_len,min_alloc_bytes); */
- if ((drv_binary = driver_alloc_binary((buf_len*5)+min_alloc_bytes))==NULL) {
- /* error handling */
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }
- drv_bin_ptr = &drv_binary;
- if ((decode_len = decode_begin(drv_bin_ptr,(unsigned char*)buf,buf_len,&err_pos)) <= ASN1_ERROR)
- {
- /* error handling due to failure in decode */
- char tmp_res_buf[5];
- driver_free_binary(*drv_bin_ptr);
- set_port_control_flags(a_data->port, 0);
-
- if(decode_len==ASN1_ERROR)
- tmp_res_buf[0]='1';
- else if(decode_len==ASN1_TAG_ERROR)
- tmp_res_buf[0]='2';
- else if(decode_len==ASN1_LEN_ERROR)
- tmp_res_buf[0]='3';
- else if(decode_len==ASN1_INDEF_LEN_ERROR)
- tmp_res_buf[0]='4';
- else if(decode_len==ASN1_VALUE_ERROR)
- tmp_res_buf[0]='5';
-/* printf("err_pos=%d\r\n",err_pos); */
-/* printf("decode_len:%d\r\n",decode_len); */
- ret_err = 1;
- while(err_pos>0){
- tmp_res_buf[ret_err] =(char)err_pos;/* c;*/
- err_pos = err_pos >> 8;
- ret_err++;
- }
- strncpy(*res_buf,tmp_res_buf,ret_err);
- return ret_err;
- }
-/* printf("decode_len=%d\r\n",decode_len); */
- if (decode_len < ((buf_len * 5) + min_alloc_bytes)) {
- /* not all memory was used => we have to reallocate */
- ErlDrvBinary *tmp;
- if ((tmp=driver_realloc_binary(*drv_bin_ptr,decode_len)) == NULL){
- /*error handling due to memory allocation failure */
- driver_free_binary(*drv_bin_ptr);
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }else
- *drv_bin_ptr=tmp;
- }
- *res_buf = (char *)(*drv_bin_ptr);
- return decode_len;
- } else { /*command == ASN1_BER_TLV_PARTIAL_DECODE */
- if ((drv_binary = driver_alloc_binary(buf_len))==NULL) {
- /* error handling */
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }
- drv_bin_ptr = &drv_binary;
- if ((decode_len = decode_partial(drv_bin_ptr,(unsigned char*)buf,buf_len))
- <= ASN1_ERROR) {
- /* error handling due to failure in decode */
- driver_free_binary(*drv_bin_ptr);
- set_port_control_flags(a_data->port, 0);
-
-/* printf("asn1_drv_control 1: decode_len=%d\r\n",decode_len); */
-
- if(decode_len==ASN1_ERROR)
- **res_buf = '1';
- return ASN1_DECODE_ERROR;
- }
- if (decode_len < buf_len) {
- /* not all memory was used => we have to reallocate */
- ErlDrvBinary *tmp;
-/* printf("asn1_drv_control 2: decode_len=%d\r\n",decode_len); */
- if ((tmp=driver_realloc_binary(*drv_bin_ptr,decode_len)) == NULL){
- /*error handling due to memory allocation failure */
- driver_free_binary(*drv_bin_ptr);
- set_port_control_flags(a_data->port, 0);
- return ASN1_MEMORY_ERROR;
- }else
- *drv_bin_ptr=tmp;
- }
- *res_buf = (char *)(*drv_bin_ptr);
- return decode_len;
- }
-}
-
-
-
-/*
- *
- * This section defines functionality for the complete encode of a
- * PER encoded message
- *
- */
-
-int complete(ErlDrvBinary **drv_binary,unsigned char *complete_buf,
- unsigned char *in_buf, int in_buf_len)
-{
- int counter = in_buf_len;
- /* counter keeps track of number of bytes left in the
- input buffer */
-
- int buf_space = in_buf_len;
- /* This is the amount of allocated space left of the complete_buf. It
- is possible when padding is applied that more space is needed than
- was originally allocated. */
-
- int buf_size = in_buf_len;
- /* Size of the buffer. May become reallocated and thus other than
- in_buf_len */
-
- unsigned char *in_ptr, *ptr;
- /* in_ptr points at the next byte in in_buf to be moved to
- complete_buf.
- ptr points into the new completed buffer, complete_buf, at the
- position of the next byte that will be set */
- int unused = 8;
- /* unused = [1,...,8] indicates how many of the rigthmost bits of
- the byte that ptr points at that are unassigned */
-
- int no_bits,no_bytes,in_unused,desired_len,ret, saved_mem, needed, pad_bits;
-
- unsigned char val;
-
- in_ptr = in_buf;
- ptr = complete_buf;
- *ptr = 0x00;
- while(counter > 0) {
- counter--;
-/* printf("*in_ptr = %d\n\r",*in_ptr); */
- switch (*in_ptr) {
- case 0:
- /* just one zero-bit should be added to the buffer */
- if(unused == 1){
- unused = 8;
- *++ptr = 0x00;
- buf_space--;
- } else
- unused--;
- break;
-
- case 1:
- /* one one-bit should be added to the buffer */
- if(unused == 1){
- *ptr = *ptr | 1;
- unused = 8;
- *++ptr = 0x00;
- buf_space--;
- } else {
- *ptr = *ptr | (1 << (unused - 1));
- unused--;
- }
- break;
-
- case 2:
- /* align buffer to end of byte */
- if (unused != 8) {
- *++ptr = 0x00;
- buf_space--;
- unused = 8;
- }
- break;
-
- case 10:
- /* next byte in in_buf tells how many bits in the second next
- byte that will be used */
- /* The leftmost unused bits in the value byte are supposed to be
- zero bits */
- no_bits = (int)*(++in_ptr);
- val = *(++in_ptr);
- counter -= 2;
- if ((ret=insert_least_sign_bits(no_bits,val,&ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 20:
- /* in this case the next value in_ptr points at holds the number
- of following bytes that holds the value that will be inserted
- in the completed buffer */
- no_bytes = (int)*(++in_ptr);
- counter -= (no_bytes + 1);
- if ((counter<0) ||
- (ret=insert_octets(no_bytes,&in_ptr,&ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 21:
- /* in this case the next two bytes in_ptr points at holds the number
- of following bytes that holds the value that will be inserted
- in the completed buffer */
- no_bytes = (int)*(++in_ptr);
- no_bytes = no_bytes << 8;
- no_bytes = no_bytes | (int)*(++in_ptr);
- counter -= (2 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets(no_bytes,&in_ptr,&ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 30:
- /* If we call the following bytes, in the buffer in_ptr points at,
- By1,By2,Rest then Rest is the value that will be transfered to
- the completed buffer. By1 tells how many of the rightmost bits in
- Rest that should not be used. By2 is the length of Rest in bytes.*/
- in_unused = (int)*(++in_ptr);
- no_bytes = (int)*(++in_ptr);
- counter -= (2 + no_bytes);
-/* printf("%d: case 30: in_unused=%d, no_bytes=%d,counter=%d\n\r",__LINE__,in_unused,no_bytes,counter); */
- ret = -4711;
- if ((counter<0) ||
- (ret=insert_octets_except_unused(no_bytes,&in_ptr,&ptr,&unused,in_unused)) == ASN1_ERROR)
- return ASN1_ERROR;
-/* printf("%d: ret=%d\n\r",__LINE__, ret); */
- buf_space -= ret;
- break;
-
- case 31:
- /* If we call the following bytes, in the buffer in_ptr points at,
- By1,By2,By3,Rest then Rest is the value that will be transfered to
- the completed buffer. By1 tells how many of the rightmost bits in
- Rest that should not be used. By2 and By3 is the length of
- Rest in bytes.*/
- in_unused = (int)*(++in_ptr);
- no_bytes = (int)*(++in_ptr);
- no_bytes = no_bytes << 8;
- no_bytes = no_bytes | (int)*(++in_ptr);
- counter -= (3 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets_except_unused(no_bytes,&in_ptr,&ptr,&unused,in_unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 40:
- /* This case implies that next byte,By1,(..,By1,By2,Bin,...)
- is the desired length of the completed value, maybe needs
- padding zero bits or removal of trailing zero bits from Bin.
- By2 is the length of Bin and Bin is the value that will be
- put into the completed buffer. Each byte in Bin has the value
- 1 or 0.*/
- desired_len = (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
-
- /* This is the algorithm for need of memory reallocation:
- Only when padding (cases 40 - 43,45 - 47) more memory may be
- used than allocated. Therefore one has to keep track of how
- much of the allocated memory that has been saved, i.e. the
- difference between the number of parsed bytes of the input buffer
- and the number of used bytes of the output buffer.
- If saved memory is less than needed for the padding then we
- need more memory. */
- saved_mem = buf_space - counter;
- pad_bits = desired_len - no_bytes - unused;
- needed = (pad_bits > 0) ? CEIL(pad_bits,8) : 0;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (2 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 41:
- /* Same as case 40 apart from By2, the length of Bin, which is in
- two bytes*/
- desired_len = (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
- no_bytes = no_bytes << 8;
- no_bytes = no_bytes | (int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (3 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 42:
- /* Same as case 40 apart from By1, the desired length, which is in
- two bytes*/
- desired_len = (int)*(++in_ptr);
- desired_len = desired_len << 8;
- desired_len = desired_len | (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (3 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 43:
- /* Same as case 40 apart from By1 and By2, the desired length and
- the length of Bin, which are in two bytes each. */
- desired_len = (int)*(++in_ptr);
- desired_len = desired_len << 8;
- desired_len = desired_len | (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
- no_bytes = no_bytes << 8;
- no_bytes = no_bytes | (int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (4 + no_bytes);
- if ((counter<0) ||
- (ret=insert_octets_as_bits_exact_len(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 45:
- /* This case assumes that the following bytes in the incoming buffer
- (called By1,By2,Bin) is By1, which is the number of bits (n) that
- will be inserted in the completed buffer. By2 is the number of
- bytes in Bin. Each bit in the buffer Bin should be inserted from
- the leftmost until the nth.*/
- desired_len = (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
-/* printf("buf_space=%d, counter=%d, needed=%d",buf_space,counter,needed); */
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (2 + no_bytes);
-/* printf("calling insert_bits_as_bits: desired_len=%d, no_bytes=%d\n\r",desired_len,no_bytes); */
-/* printf("1in_ptr=%d\n\r",in_ptr); */
-
- if((counter<0) ||
- (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
-/* printf("2in_ptr=%d, ptr=%d, complete_buf=%d\n\r",in_ptr,ptr,complete_buf); */
-/* printf("buf_space=%d, ret=%d, counter=%d\n\r",buf_space,ret,counter); */
- buf_space -= ret;
- break;
-
- case 46:
- /* Same as case 45 apart from By1, the desired length, which is
- in two bytes. */
- desired_len = (int)*(++in_ptr);
- desired_len = desired_len << 8;
- desired_len = desired_len | (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (3 + no_bytes);
- if((counter<0) ||
- (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- case 47:
- /* Same as case 45 apart from By1 and By2, the desired length
- and the length of Bin, which are in two bytes each. */
- desired_len = (int)*(++in_ptr);
- desired_len = desired_len << 8;
- desired_len = desired_len | (int)*(++in_ptr);
- no_bytes=(int)*(++in_ptr);
- no_bytes = no_bytes << 8;
- no_bytes = no_bytes | (int)*(++in_ptr);
-
- saved_mem = buf_space - counter;
- needed = CEIL((desired_len-unused),8) - no_bytes;
- if (saved_mem < needed) {
- /* Have to allocate more memory */
- buf_size += needed;
- buf_space += needed;
- if (realloc_memory(drv_binary,buf_size,&ptr,
- &complete_buf) == ASN1_ERROR)
- return ASN1_ERROR;
- }
-
- counter -= (4 + no_bytes);
- if((counter<0) ||
- (ret=insert_bits_as_bits(desired_len,no_bytes,&in_ptr,
- &ptr,&unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- buf_space -= ret;
- break;
-
- default:
- return ASN1_ERROR;
- }
- in_ptr++;
- }
- /* The returned buffer must be at least one byte and
- it must be octet aligned */
- if ((unused == 8) && (ptr != complete_buf))
- return (ptr - complete_buf);
- else {
- ptr++; /* octet align buffer */
- return (ptr - complete_buf);
- }
-}
-
-
-int realloc_memory(ErlDrvBinary **drv_binary,
- int amount,
- unsigned char **ptr,
- unsigned char **complete_buf) {
-
- ErlDrvBinary *tmp_bin;
- int i;
-
-/* printf("realloc_momory: amount = %d\n",amount); */
- if ((tmp_bin=driver_realloc_binary(*drv_binary,amount)) == NULL) {
- /*error handling due to memory allocation failure */
-/* printf("error when allocating memory\n"); */
- return ASN1_ERROR;
- }else {
- i = *ptr - *complete_buf;
- *drv_binary=tmp_bin;
- *complete_buf = (unsigned char*)(*drv_binary)->orig_bytes;
- *ptr = *complete_buf + i;
- }
- return ASN1_OK;
-}
-
-
-int insert_most_sign_bits(int no_bits,
- unsigned char val,
- unsigned char **output_ptr,
- int *unused) {
- unsigned char *ptr = *output_ptr;
-
- if (no_bits < *unused){
- *ptr = *ptr | (val >> (8 - *unused));
- *unused -= no_bits;
- } else if (no_bits == *unused) {
- *ptr = *ptr | (val >> (8 - *unused));
- *unused = 8;
- *++ptr = 0x00;
- } else {
- *ptr = *ptr | (val >> (8 - *unused));
- *++ptr = 0x00;
- *ptr = *ptr | (val << *unused);
- *unused = 8 - (no_bits - *unused);
- }
- *output_ptr = ptr;
- return ASN1_OK;
-}
-
-
-int insert_least_sign_bits(int no_bits,
- unsigned char val,
- unsigned char **output_ptr,
- int *unused) {
- unsigned char *ptr = *output_ptr;
- int ret = 0;
-
- if (no_bits < *unused){
- *ptr = *ptr | (val << (*unused - no_bits));
- *unused -= no_bits;
- } else if (no_bits == *unused){
- *ptr = *ptr | val;
- *unused = 8;
- *++ptr = 0x00;
- ret++;
- } else {
- /* first in the begun byte in the completed buffer insert
- so many bits that fit, then insert the rest in next byte.*/
- *ptr = *ptr | (val >> (no_bits - *unused));
- *++ptr = 0x00;
- ret++;
- *ptr = *ptr | (val << (8 - (no_bits - *unused)));
- *unused = 8 - (no_bits - *unused);
- }
- *output_ptr = ptr;
- return ret;
-}
-
-/* pad_bits adds no_bits bits in the buffer that output_ptr
- points at.
- */
-int pad_bits(int no_bits, unsigned char **output_ptr, int *unused)
- {
- unsigned char *ptr = *output_ptr;
- int ret = 0;
-
- while (no_bits > 0) {
- if(*unused == 1){
- *unused = 8;
- *++ptr = 0x00;
- ret++;
- } else
- (*unused)--;
- no_bits--;
- }
- *output_ptr = ptr;
- return ret;
- }
-
-
-/* insert_bits_as_bits removes no_bytes bytes from the buffer that in_ptr
- points at and takes the desired_no leftmost bits from those removed
- bytes and inserts them in the buffer(output buffer) that ptr points at.
- The unused parameter tells how many bits that are not set in the
- actual byte in the output buffer. If desired_no is more bits than the
- input buffer has in no_bytes bytes, then zero bits is padded.*/
-int insert_bits_as_bits(int desired_no,
- int no_bytes,
- unsigned char **input_ptr,
- unsigned char **output_ptr,
- int *unused)
-{
- unsigned char *in_ptr = *input_ptr;
- unsigned char val;
- int no_bits, ret, ret2;
-
- if (desired_no == (no_bytes * 8)) {
- if(insert_octets_unaligned(no_bytes,&in_ptr,output_ptr,*unused)
- == ASN1_ERROR)
- return ASN1_ERROR;
- ret = no_bytes;
- }
- else if (desired_no < (no_bytes * 8)) {
-/* printf("insert_bits_as_bits 1\n\r"); */
- if(insert_octets_unaligned(desired_no/8,&in_ptr,output_ptr,*unused)
- == ASN1_ERROR)
- return ASN1_ERROR;
-/* printf("insert_bits_as_bits 2\n\r"); */
- val = *++in_ptr;
-/* printf("val = %d\n\r",(int)val); */
- no_bits = desired_no % 8;
-/* printf("no_bits = %d\n\r",no_bits); */
- insert_most_sign_bits(no_bits,val,output_ptr,unused);
- ret = CEIL(desired_no,8);
- }
- else {
- if(insert_octets_unaligned(no_bytes,&in_ptr,output_ptr,*unused)
- == ASN1_ERROR)
- return ASN1_ERROR;
- ret2 = pad_bits(desired_no - (no_bytes * 8),output_ptr,unused);
-/* printf("ret2 = %d\n\r",ret2); */
- ret = CEIL(desired_no,8);
-/* printf("ret = %d\n\r",ret); */
- }
-/* printf("*unused = %d\n\r",*unused); */
- *input_ptr = in_ptr;
- return ret;
-}
-
-
-/* insert_octets_as_bits_exact_len */
-int
-insert_octets_as_bits_exact_len(int desired_len,
- int in_buff_len,
- unsigned char **in_ptr,
- unsigned char **ptr,
- int *unused)
-{
- int ret = 0;
- int ret2 = 0;
-
- if (desired_len == in_buff_len) {
- if ((ret = insert_octets_as_bits(in_buff_len,in_ptr,ptr,unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- }
- else if(desired_len > in_buff_len) {
- if((ret = insert_octets_as_bits(in_buff_len,in_ptr,ptr,unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- /* now pad with zero bits */
-/* printf("~npad_bits: called with %d bits padding~n~n~r",desired_len - in_buff_len); */
- if ((ret2=pad_bits(desired_len - in_buff_len,ptr,unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- }
- else {/* desired_len < no_bits */
- if ((ret=insert_octets_as_bits(desired_len,in_ptr,ptr,unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- /* now remove no_bits - desired_len bytes from in buffer */
- *in_ptr += (in_buff_len - desired_len);
- }
- return (ret+ret2);
-}
-
-
-
-/* insert_octets_as_bits takes no_bytes bytes from the buffer that input_ptr
- points at and inserts the least significant bit of it in the buffer that
- output_ptr points at. Each byte in the input buffer must be 1 or 0
- otherwise the function returns ASN1_ERROR. The output buffer is concatenated
- without alignment.
- */
-int insert_octets_as_bits(int no_bytes,
- unsigned char **input_ptr,
- unsigned char **output_ptr,
- int *unused)
-{
- unsigned char *in_ptr = *input_ptr;
- unsigned char *ptr = *output_ptr;
- int used_bits = 8 - *unused;
-
- while (no_bytes > 0) {
- switch (*++in_ptr) {
- case 0:
- if(*unused == 1){
- *unused = 8;
- *++ptr = 0x00;
- } else
- (*unused)--;
- break;
- case 1:
- if(*unused == 1){
- *ptr = *ptr | 1;
- *unused = 8;
- *++ptr = 0x00;
- } else {
- *ptr = *ptr | (1 << (*unused - 1));
- (*unused)--;
- }
- break;
- default:
- return ASN1_ERROR;
- }
- no_bytes--;
- }
- *input_ptr = in_ptr;
- *output_ptr = ptr;
- return ((used_bits+no_bytes) / 8); /*return number of new bytes
- in completed buffer */
-}
-
-/* insert_octets inserts bytes from the input buffer, *input_ptr,
- into the output buffer, *output_ptr. Before the first byte is
- inserted the input buffer is aligned.
- */
-int insert_octets(int no_bytes,
- unsigned char **input_ptr,
- unsigned char **output_ptr,
- int *unused)
-{
- unsigned char *in_ptr = *input_ptr;
- unsigned char *ptr = *output_ptr;
- int ret = 0;
-
- if (*unused != 8) {/* must align before octets are added */
- *++ptr = 0x00;
- ret++;
- *unused = 8;
- }
- while(no_bytes > 0) {
- *ptr = *(++in_ptr);
- *++ptr = 0x00;
- /* *unused = *unused - 1; */
- no_bytes--;
- }
- *input_ptr = in_ptr;
- *output_ptr = ptr;
- return (ret + no_bytes);
-}
-
-/* insert_octets_unaligned inserts bytes from the input buffer, *input_ptr,
- into the output buffer, *output_ptr.No alignment is done.
- */
-int insert_octets_unaligned(int no_bytes,
- unsigned char **input_ptr,
- unsigned char **output_ptr,
- int unused)
-{
- unsigned char *in_ptr = *input_ptr;
- unsigned char *ptr = *output_ptr;
- int n = no_bytes;
- unsigned char val;
-
- while (n > 0) {
- if (unused == 8) {
- *ptr = *++in_ptr;
- *++ptr = 0x00;
- }else {
- val = *++in_ptr;
- *ptr = *ptr | val >> (8 - unused);
- *++ptr = 0x00;
- *ptr = val << unused;
- }
- n--;
- }
- *input_ptr = in_ptr;
- *output_ptr = ptr;
- return no_bytes;
-}
-
-
-int insert_octets_except_unused(int no_bytes,
- unsigned char **input_ptr,
- unsigned char **output_ptr,
- int *unused,
- int in_unused)
-{
- unsigned char *in_ptr = *input_ptr;
- unsigned char *ptr = *output_ptr;
- int val, no_bits;
- int ret = 0;
-
- if (in_unused == 0){
-/* printf("%d: insert_octets_except_unused: if\n\r",__LINE__); */
- if ((ret = insert_octets_unaligned(no_bytes,&in_ptr,&ptr,
- *unused)) == ASN1_ERROR)
- return ASN1_ERROR;
- }
- else {
-/* printf("%d: insert_octets_except_unused: else\n\r",__LINE__); */
- if ((ret=insert_octets_unaligned(no_bytes - 1,&in_ptr,&ptr,*unused)) != ASN1_ERROR) {
- val = (int) *(++in_ptr);
- no_bits = 8 - in_unused;
- /* no_bits is always less than *unused since the buffer is
- octet aligned after insert:octets call, so the following
- if clasuse is obsolete I think */
- if(no_bits < *unused){
- *ptr = *ptr | (val >> (8 - *unused));
- *unused = *unused - no_bits;
- } else if (no_bits == *unused) {
- *ptr = *ptr | (val >> (8 - *unused));
- *++ptr = 0x00;
- ret++;
- *unused = 8;
- } else {
- *ptr = *ptr | (val >> (8 - *unused));
- *++ptr = 0x00;
- ret++;
- *ptr = *ptr | (val << *unused);
- *unused = 8 - (no_bits - *unused);
- }
- } else
- return ASN1_ERROR;
- }
- *input_ptr = in_ptr;
- *output_ptr = ptr;
-/* printf("%d: insert_octets_except_unused: ret=%d\n\r",__LINE__,ret); */
- return ret;
-}
-
-
-
-/*
- *
- * This section defines functionality for the partial decode of a
- * BER encoded message
- *
- */
-
-/*
- * int decode(ErlDrvBinary **drv_binary,unsigned char *decode_buf,
- * unsigned char *in_buf, int in_buf_len)
- * drv_binary is a pointer to a pointer to an allocated driver binary.
- * in_buf is a pointer into the buffer of incoming bytes.
- * in_buf_len is the length of the incoming buffer.
- * The function reads the bytes in the incoming buffer and structures
- * it in a nested way as Erlang terms. The buffer contains data in the
- * order tag - length - value. Tag, length and value has the following
- * format:
- * A tag is normally one byte but may be of any length, if the tag number
- * is greater than 30. +----------+
- * |CL|C|NNNNN|
- * +----------+
- * If NNNNN is 31 then will the 7 l.s.b of each of the following tag number
- * bytes contain the tag number. Each tag number byte that is not the last one
- * has the m.s.b. set to 1.
- * The length can be short definite length (sdl), long definite length (ldl)
- * or indefinite length (il).
- * sdl: +---------+ the L bits is the length
- * |0|LLLLLLL|
- * +---------+
- * ldl: +---------+ +---------+ +---------+ +-----------+
- * |1|lllllll| |first len| | | |the Nth len|
- * +---------+ +---------+ +---------+ ... +-----------+
- * The first byte tells how many len octets will follow, max 127
- * il: +---------+ +----------------------+ +--------+ +--------+
- * |1|0000000| |content octets (Value)| |00000000| |00000000|
- * +---------+ +----------------------+ +--------+ +--------+
- * The value octets are preceded by one octet and followed by two
- * exactly as above. The value must be some tag-length-value encoding.
- *
- * The function returns a value in Erlnag term format:
- * {{TagNo,Value},Rest}
- * TagNo is an integer ((CL bsl 16) + tag number) which limits the tag number
- * to 65535.
- * Value is a binary if the C bit in tag was unset, otherwise (if tag was
- * constructed) Value is a list, List.
- * List is like: [{TagNo,Value},{TagNo,Value},...]
- * Rest is a binary, i.e. the undecoded part of the buffer. Most often Rest
- * is the empty binary.
- * If some error occured during the decoding of the in_buf an error is returned.
- */
-int decode_begin(ErlDrvBinary **drv_binary,unsigned char *in_buf, int in_buf_len, unsigned int *err_pos)
-{
- int maybe_ret;
- char *decode_buf = (*drv_binary)->orig_bytes;
- int ei_index = 0;
- int ib_index = 0;
- /* ei_index is the index used by the ei functions to encode an
- Erlang term into the buffer decode_buf */
- /* ib_index is the index were to read the next byte from in_buf */
-
-
-#ifdef ASN1_DEBUG
- printf("decode_begin1: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index);
-#endif
- /* the first byte must be a "version magic" */
- if(ei_encode_version(decode_buf,&ei_index) == ASN1_ERROR)
- return ASN1_ERROR; /* 1 byte */
-#ifdef ASN1_DEBUG
- printf("decode_begin2: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index);
-#endif
- if (ei_encode_tuple_header(decode_buf,&ei_index,2) == ASN1_ERROR)
- return ASN1_ERROR; /* 2 bytes */
-#ifdef ASN1_DEBUG
- printf("decode_begin3: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index);
-#endif
- if((maybe_ret=decode(drv_binary,&ei_index,in_buf,&ib_index,in_buf_len)) <= ASN1_ERROR)
- {
- *err_pos = ib_index;
-#ifdef ASN1_DEBUG
- printf("err_pos=%d,ib_index=%d\r\n",*err_pos,ib_index);
-#endif
- return maybe_ret;
- };
-
- decode_buf = (*drv_binary)->orig_bytes; /* maybe a realloc during decode_value */
-#ifdef ASN1_DEBUG
- printf("decode_begin4: in_buf_len=%d, ei_index=%d, ib_index=%d\n\r",
- in_buf_len,ei_index,ib_index);
-#endif
- /* "{{TagNo,Value},Rest}" */
- if (ei_encode_binary(decode_buf,&ei_index,&(in_buf[ib_index]),in_buf_len-ib_index)
- == ASN1_ERROR) /* at least 5 bytes */
- return ASN1_ERROR;
-#ifdef ASN1_DEBUG
- printf("decode_begin5: ei_index=%d, ib_index=%d\n\r",ei_index,ib_index);
-#endif
- return ei_index;
-}
-
-int decode(ErlDrvBinary **drv_binary,int *ei_index,unsigned char *in_buf,
- int *ib_index, int in_buf_len)
-{
- int maybe_ret;
- char *decode_buf = (*drv_binary)->orig_bytes;
- int form;
-#ifdef ASN1_DEBUG
- printf("decode 1\n\r");
-#endif
- if (((*drv_binary)->orig_size - *ei_index) < 19) {/* minimum amount of bytes */
- /* allocate more memory */
- if (realloc_decode_buf(drv_binary,(*drv_binary)->orig_size * 2) ==
- ASN1_ERROR)
- return ASN1_ERROR;
- decode_buf = (*drv_binary)->orig_bytes;
- }
-/* printf("decode 2\n\r"); */
- /* "{" */
- if (ei_encode_tuple_header(decode_buf,ei_index,2) == ASN1_ERROR)
- return ASN1_ERROR; /* 2 bytes */
-#ifdef ASN1_DEBUG
- printf("decode 3:orig_size=%ld, ei_index=%d, ib_index=%d\n\r",(*drv_binary)->orig_size,*ei_index,*ib_index);
-#endif
-
- /*buffer must hold at least two bytes*/
- if ((*ib_index +2) > in_buf_len)
- return ASN1_VALUE_ERROR;
- /* "{{TagNo," */
- if ((form = decode_tag(decode_buf,ei_index,in_buf,in_buf_len,ib_index)) <= ASN1_ERROR)
- return form; /* 5 bytes */
-#ifdef ASN1_DEBUG
- printf("i_i=%d,in_buf_len=%d\r\n",*ei_index,in_buf_len);
-#endif
- if (*ib_index >= in_buf_len){
- return ASN1_TAG_ERROR;
- }
-#ifdef ASN1_DEBUG
- printf("decode 5 ib_index=%d\n\r",*ib_index);
-#endif
- /* buffer must hold at least one byte (0 as length and nothing as
- value) */
- /* "{{TagNo,Value}," */
- if ((maybe_ret=decode_value(ei_index,in_buf,ib_index,drv_binary,form,
- in_buf_len)) <= ASN1_ERROR)
- return maybe_ret; /* at least 5 bytes */
-#ifdef ASN1_DEBUG
- printf("decode 7\n\r");
-#endif
- return *ei_index;
-}
-
-/*
- * decode_tag decodes the BER encoded tag in in_buf and puts it in the
- * decode_buf encoded by the Erlang extern format as an Erlang term.
- */
-int decode_tag(char *decode_buf,int *db_index,unsigned char *in_buf,
- int in_buf_len, int *ib_index)
-{
- int tag_no, tmp_tag, form;
-
-
- /* first get the class of tag and bit shift left 16*/
- tag_no = ((MASK(in_buf[*ib_index],ASN1_CLASS)) << 10);
-
- form = (MASK(in_buf[*ib_index],ASN1_FORM));
-#ifdef ASN1_DEBUG
- printf("decode_tag0:ii=%d, tag_no=%d, form=%d.\r\n",
- *ib_index,tag_no,form);
-#endif
-
- /* then get the tag number */
- if((tmp_tag = (int) INVMASK(in_buf[*ib_index],ASN1_CLASSFORM)) < 31) {
- ei_encode_ulong(decode_buf,db_index,tag_no+tmp_tag); /* usual case */
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_tag1:ii=%d.\r\n",*ib_index);
-#endif
- }
- else
- {
- int n = 0; /* n is used to check that the 64K limit is not
- exceeded*/
-#ifdef ASN1_DEBUG
- printf("decode_tag1:ii=%d, in_buf_len=%d.\r\n",*ib_index,in_buf_len);
-#endif
-
- /* should check that at least three bytes are left in
- in-buffer,at least two tag byte and at least one length byte */
- if ((*ib_index +3) > in_buf_len)
- return ASN1_VALUE_ERROR;
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_tag2:ii=%d.\r\n",*ib_index);
-#endif
- /* The tag is in the following bytes in in_buf as
- 1ttttttt 1ttttttt ... 0ttttttt, where the t-bits
- is the tag number*/
- /* In practice is the tag size limited to 64K, i.e. 16 bits. If
- the tag is greater then 64K return an error */
- while (((tmp_tag = (int)in_buf[*ib_index]) >= 128) && n < 2){
- /* m.s.b. = 1 */
- tag_no = tag_no + (MASK(tmp_tag,ASN1_LONG_TAG) << 7);
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_tag3:ii=%d.\r\n",*ib_index);
-#endif
- n++;
- };
- if ((n==2) && in_buf[*ib_index] > 3)
- return ASN1_TAG_ERROR; /* tag number > 64K */
- tag_no = tag_no + in_buf[*ib_index];
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_tag4:ii=%d.\r\n",*ib_index);
-#endif
- ei_encode_ulong(decode_buf,db_index,tag_no);
- }
- return form;
-}
-
-
-/*
- * decode_value decodes the BER encoded length and value fields in the
- * in_buf and puts the value part in the decode_buf as an Erlang term
- * encoded by the Erlang extern format
- */
-int decode_value(int *ei_index,unsigned char *in_buf,
- int *ib_index,ErlDrvBinary **drv_binary,int form,
- int in_buf_len)
-{
- int maybe_ret;
- char *decode_buf = (*drv_binary)->orig_bytes;
- unsigned int len = 0;
- unsigned int lenoflen = 0;
- int indef = 0;
-
-#ifdef ASN1_DEBUG
- printf("decode_value1:ib_index=%d\n\r",*ib_index);
-#endif
- if (((in_buf[*ib_index]) & 0x80) == ASN1_SHORT_DEFINITE_LENGTH) {
- len = in_buf[*ib_index];
- }
- else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH)
- indef = 1;
- else /* long definite length */ {
- lenoflen = (in_buf[*ib_index] & 0x7f); /*length of length */
-#ifdef ASN1_DEBUG
- printf("decode_value,lenoflen:%d\r\n",lenoflen);
-#endif
- if (lenoflen > (in_buf_len - (*ib_index+1)))
- return ASN1_LEN_ERROR;
- len = 0;
- while (lenoflen-- ) {
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_value1:*ib_index=%d, byte = %d.\r\n",*ib_index,in_buf[*ib_index]);
-#endif
- if (!(len < (1 << (sizeof(len)-1)*8)))
- return ASN1_LEN_ERROR; /* length does not fit in 32 bits */
- len = (len << 8) + in_buf[*ib_index];
- }
- }
- if (len > (in_buf_len - (*ib_index + 1)))
- return ASN1_VALUE_ERROR;
- (*ib_index)++;
-#ifdef ASN1_DEBUG
- printf("decode_value2:ii=%d.\r\n",*ib_index);
-#endif
- if (indef == 1)
- { /* in this case it is desireably to check that indefinite length
- end bytes exist in inbuffer */
- while (!(in_buf[*ib_index]==0 && in_buf[*ib_index + 1]==0)) {
-#ifdef ASN1_DEBUG
- printf("decode_value while:ib_index=%d in_buf_len=%d\n\r",
- *ib_index,in_buf_len);
-#endif
- if(*ib_index >= in_buf_len)
- return ASN1_INDEF_LEN_ERROR;
- ei_encode_list_header(decode_buf,ei_index,1); /* 5 bytes */
- if((maybe_ret=decode(drv_binary,ei_index,in_buf,
- ib_index,in_buf_len)) <= ASN1_ERROR)
- return maybe_ret;
- decode_buf = (*drv_binary)->orig_bytes;
- }
- (*ib_index) += 2; /* skip the indefinite length end bytes */
-#ifdef ASN1_DEBUG
- printf("decode_value3:ii=%d.\r\n",*ib_index);
-#endif
- ei_encode_empty_list(decode_buf,ei_index); /* 1 byte */
- }
- else if (form == ASN1_CONSTRUCTED)
- {
- int end_index = *ib_index + len;
- if(end_index > in_buf_len)
- return ASN1_LEN_ERROR;
- while (*ib_index < end_index) {
-
-#ifdef ASN1_DEBUG
- printf("decode_value3:*ib_index=%d, end_index=%d\n\r",*ib_index,end_index);
-#endif
- ei_encode_list_header(decode_buf,ei_index,1); /* 5 bytes */
- if((maybe_ret=decode(drv_binary,ei_index,in_buf,
- ib_index,in_buf_len))<=ASN1_ERROR)
- return maybe_ret;
- decode_buf = (*drv_binary)->orig_bytes;
- }
- ei_encode_empty_list(decode_buf,ei_index); /* 1 byte */
- }
- else
- {
- if (((*drv_binary)->orig_size - *ei_index) < 10+len) { /* 5+len for the binary*/
- if (realloc_decode_buf(drv_binary,(*drv_binary)->orig_size * 2) ==
- ASN1_ERROR)
- return ASN1_ERROR;
- decode_buf = (*drv_binary)->orig_bytes;
- }
- if((*ib_index + len) > in_buf_len)
- return ASN1_LEN_ERROR;
- ei_encode_binary(decode_buf,ei_index,&in_buf[*ib_index],len);
- *ib_index = *ib_index + len;
-#ifdef ASN1_DEBUG
- printf("decode_value4:ii=%d.\r\n",*ib_index);
-#endif
- }
- return ASN1_OK;
-}
-
-int realloc_decode_buf(ErlDrvBinary **drv_binary,int amount) {
- ErlDrvBinary *tmp_bin;
-
- if ((tmp_bin=driver_realloc_binary(*drv_binary,amount)) == NULL)
- return ASN1_ERROR;
- *drv_binary = tmp_bin;
- return ASN1_OK;
-}
-
-
-
-/*
- * int decode_partial(drv_binary,in_buf,in_buf_len)
- */
-/*
- * The in_buf contains two parts: first information about which value
- * will be decoded, as a sequence of tags and tag codes, then the
- * encoded BER value. First of all comes a length field that tells how
- * many following bytes contains the sequence of tags. Then starts the
- * BER encoded message. The tag sequence length field is a single
- * byte. The sequence of tags/tag codes may be one of the codes
- * ASN1_SKIPPED, ASN1_CHOOSEN and a tag or ASN1_OPTIONAL and a
- * tag. ASN1_SKIPPED means that the following tag is mandatory and is
- * skipped. ASN1_CHOOSEN means that the value of this tag shall, if
- * this was the last tag in tag sequence, be returned or be searched
- * in for the next tag. ASN1_OPTIONAL means that this tag shall be
- * skipped but it may be missing. Each tag in the tag sequence
- * correspond to a tag in the BER encoded message. If the decode
- * arives to a position where there is no matching tag, an error is
- * returned (if it wasn't the last tag and it was OPTIONAL). After the
- * right value has been detected it is returned in the out_buf.
- *
- */
-int decode_partial(ErlDrvBinary **drv_binary,unsigned char *in_buf, int in_buf_len)
-{
- char *out_buf = (*drv_binary)->orig_bytes;
- int tag_index_val = 1;
- int msg_index_val;
- int *msg_index, *tag_index, tmp_index;
- int tag_seq_length;
- int wanted_tag, next_tag;
- int buf_end_index = in_buf_len;
- int ret = 0, length, old_index;
-
- tag_index = &tag_index_val;
- tag_seq_length = in_buf[0];
- msg_index = &msg_index_val;
- *msg_index = tag_seq_length + 1;
-
-
-/* printf("decode_partial 1: in_buf_len=%d, tag_index=%d, msg_index=%d\r\n,tag_seq_length=%d\r\n",in_buf_len,*tag_index,*msg_index,tag_seq_length); */
- while(*tag_index < tag_seq_length) {
- switch(in_buf[*tag_index]) {
- case ASN1_SKIPPED:
-/* printf("decode_partial ASN1_SKIPPED: in_buf[*msg_index]=%d\r\n",in_buf[*msg_index]); */
- (*tag_index)++;
-/* printf("decode_partial ASN1_SKIPPED 2: *msg_index=%d\r\n",*msg_index); */
- skip_tag(in_buf,msg_index,buf_end_index);
-/* printf("decode_partial ASN1_SKIPPED 3: *msg_index=%d\r\n",*msg_index); */
- skip_length_and_value(in_buf,msg_index,buf_end_index);
-/* printf("decode_partial ASN1_SKIPPED 4: *msg_index=%d\r\n",*msg_index); */
- break;
- case ASN1_OPTIONAL:
- (*tag_index)++;
-/* printf("decode_partial ASN1_OPTIONAL: in_buf[*tag_index]=%d\r\n",in_buf[*tag_index]); */
- wanted_tag = in_buf[*tag_index];
- (*tag_index)++;
- tmp_index = *msg_index;
- next_tag = get_tag(in_buf,msg_index,buf_end_index);
- if (wanted_tag != next_tag) {
- *msg_index = tmp_index;
- } else
- skip_length_and_value(in_buf,msg_index,buf_end_index);
- break;
- case ASN1_CHOOSEN:
-/* printf("decode_partial ASN1_CHOOSEN: in_buf[*msg_index]=%d, *msg_index=%d\r\n",in_buf[*msg_index],*msg_index); */
- (*tag_index)++;
- wanted_tag = in_buf[*tag_index];
- (*tag_index)++;
- old_index = *msg_index;
-/* printf("decode_partial ASN1_CHOOSEN 2: *msg_index=%d\r\n",*msg_index); */
- next_tag = get_tag(in_buf,msg_index,buf_end_index);
-/* printf("decode_partial ASN1_CHOOSEN 3: *msg_index=%d\r\n,wanted_tag=%d, next_tag=%d\r\n",*msg_index,wanted_tag,next_tag); */
- if (wanted_tag != next_tag)
- return ASN1_NOVALUE; /* an empty binary will be returned to Erlang */
- if (*tag_index == (tag_seq_length + 1)) {
- /* get the value and return*/
- if((ret = get_value(out_buf,in_buf,msg_index,buf_end_index)) <= ASN1_ERROR)
- return ASN1_ERROR;
- return ret;
- }
- else {
- /* calculate the length of the sub buffer and let *msg_index
- be at the value part of this BER encoded type*/
- int indef;
- indef = 0;
- length = get_length(in_buf,msg_index,&indef,buf_end_index);
-/* printf("decode_partial ASN1_CHOOSEN 4: length=%d, *msg_index=%d\r\n",length,*msg_index); */
- if ((length == 0) && (indef == 1)) {
- /* indefinite length of value */
- old_index = *msg_index;
- length = skip_length_and_value(in_buf,msg_index,buf_end_index);
- *msg_index = old_index;
- buf_end_index = *msg_index + length - 2;
- /* remove two bytes due to indefinete length end zeros */
- } else
- buf_end_index = (*msg_index + length);
- }
- break;
- default:
- return ASN1_ERROR;
- }
- }
- return ASN1_ERROR;
-}
-
-
-/*
- * int skip_tag(unsigned char *in_buf,int *index,int buf_len)
- * steps past the BER encoded tag in in_buf and updates *index.
- * Returns the number of skipped bytes.
- */
-int skip_tag(unsigned char *in_buf,int *index,int buf_len)
-{
- int start_index = *index;
- if ((MASK(in_buf[*index],ASN1_TAG)) == 31){
- do {
- (*index)++;
- if (*index >= buf_len)
- return ASN1_ERROR;
- }
- while(in_buf[*index] >=128);
- }
- (*index)++;
- return (*index - start_index);
-}
-
-
-/*
- * int skip_length_and_value(unsigned char *in_buf,int *index,int buf_len)
- * steps past the BER encoded length and value in in_buf and updates *index.
- * returns the length if the skipped "length value".
- * Returns the number of skipped bytes.
- */
-int skip_length_and_value(unsigned char *in_buf,int *index,int buf_len)
-{
- long len;
- int indef = 0, lenoflen;
- int start_index = *index;
-
- if ((MASK(in_buf[*index],0x80)) == ASN1_SHORT_DEFINITE_LENGTH){
- len = in_buf[*index];
- if (len > (buf_len - (*index + 1)))
- return ASN1_LEN_ERROR;
- } else if (in_buf[*index] == ASN1_INDEFINITE_LENGTH)
- indef = 1;
- else /* long definite length */ {
- lenoflen = (in_buf[*index] & 0x7f); /*length of length */
- len = 0;
- while (lenoflen--) {
- (*index)++;
- len = (len << 8) + in_buf[*index];
- }
- if (len > (buf_len - (*index + 1)))
- return ASN1_LEN_ERROR;
- }
- (*index)++;
- if (indef == 1)
- {
- while(!(in_buf[*index]==0 && in_buf[*index + 1]==0)) {
- skip_tag(in_buf,index,buf_len);
- skip_length_and_value(in_buf,index,buf_len);
- }
- (*index) += 2;
- }
- else
- (*index) += len;
- return (*index - start_index);
-}
-
-/* int get_tag(unsigned char *in_buf,int *index)
- *
- * assumes next byte/bytes in in_buf is an encoded BER tag. A tag
- * number has theoretically no upper limit in size. Here the tag
- * number is assumed to be less than 64K. Returns an integer value
- * on the format:
- * xxxxxxxx xxxxxxcc tttttttt tttttttt
- * the x-bits are 0 (insignificant)
- * the c-bits are the class of the tag
- * the t-bits are the tag number. This implies that the tag number
- * is limited to 64K-1
- *
- */
-int get_tag(unsigned char *in_buf,int *index,int buf_len)
-{
- int tag_no = 0,tmp_tag = 0;
-
- tag_no = (MASK(in_buf[*index],ASN1_CLASSFORM));
- if ((MASK(in_buf[*index],ASN1_TAG)) == ASN1_TAG) {
- /* long form of tag */
- do {
- (*index)++;
- if (*index >= buf_len)
- return ASN1_TAG_ERROR;
- tmp_tag = tmp_tag << 7;
- tmp_tag += (MASK(in_buf[*index],ASN1_LONG_TAG));
- } while (in_buf[*index] >= 128);
- (*index)++;
- tag_no = tag_no + tmp_tag;
- } else {
- tag_no += (MASK(in_buf[*index],ASN1_TAG));
- (*index)++;
- }
- if (*index >= buf_len)
- return ASN1_TAG_ERROR;
- return tag_no;
-}
-
-
-/*
- * int get_value(char *out_buf,unsigned char *in_buf,
- * int *msg_index,int in_buf_len)
- */
-/* assumes next byte/bytes in in_buf is an encoded BER value preceeded by a BER encoded length. Puts value in out_buf.
- */
-int get_value(char *out_buf,
- unsigned char *in_buf,
- int *msg_index,
- int in_buf_len)
-{
- int len, lenoflen, indef=0, skip_len;
- int ret=0;
- int start_index;
-
-/* printf("get_value 1\n\r"); */
- if (in_buf[*msg_index] < 0x80){ /* short definite length */
- len = in_buf[*msg_index];
-/* printf("short definite length\r\n"); */
- } else if (in_buf[*msg_index] > 0x80) { /* long definite length */
- lenoflen = (in_buf[*msg_index] & 0x7f); /*length of length */
- len = 0;
- while (lenoflen--) {
- (*msg_index)++;
- len = (len << 8) + in_buf[*msg_index];
- }
- if (len > (in_buf_len - (*msg_index + 1)))
- return ASN1_LEN_ERROR;
- } else
- indef = 1;
- (*msg_index)++;
-/* printf("get_value 2: len = %d, *msg_index = %d\r\n",len,*msg_index); */
- if (indef == 1) {
- while(!(in_buf[*msg_index]==0 && in_buf[*msg_index + 1]==0)) {
- start_index = *msg_index;
- skip_len = skip_tag(in_buf,msg_index,in_buf_len);
-/* printf("get_value 3: skip_len=%d,start_index=%d,*msg_index=%d\n\r", */
-/* skip_len,start_index,*msg_index); */
- memcpy(&out_buf[ret],&in_buf[start_index],skip_len);
- ret += skip_len;
- start_index = *msg_index;
- skip_len = skip_length_and_value(in_buf,msg_index,in_buf_len);
-/* printf("get_value 4: skip_len=%d,start_index=%d,*msg_index=%d\n\r", */
-/* skip_len,start_index,*msg_index); */
- memcpy(&out_buf[ret],&in_buf[start_index],skip_len);
- ret += skip_len;
- }
- return ret;
- }
- else
- memcpy(&out_buf[ret],&in_buf[*msg_index],len);
- return len;
-}
-
-
-/*
- * int get_length(unsigned char *in_buf,int *msg_index)
- * assumes next byte/bytes contain a BER encoded length field,
- * which is decoded. The value of the length is returned. If it
- * is an indefinite length the *indef is set to one.
- */
-int get_length(unsigned char *in_buf,int *msg_index,
- int *indef,int in_buf_len)
-{
- int len=0, lenoflen;
-
- if (in_buf[*msg_index] < 0x80) /* short definite length */
- len = in_buf[*msg_index];
- else if (in_buf[*msg_index] > 0x80) { /* long definite length */
- lenoflen = (in_buf[*msg_index] & 0x7f); /*length of length */
- len = 0;
- while (lenoflen--) {
- (*msg_index)++;
- len = (len << 8) + in_buf[*msg_index];
- }
- if (len > (in_buf_len - (*msg_index + 1)))
- return ASN1_LEN_ERROR;
- } else
- *indef = 1;
- (*msg_index)++;
- return len;
-}
diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c
new file mode 100644
index 0000000000..58314e23c2
--- /dev/null
+++ b/lib/asn1/c_src/asn1_erl_nif.c
@@ -0,0 +1,1079 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ *
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "erl_nif.h"
+
+/* #define ASN1_DEBUG 1 */
+
+#define ASN1_OK 0
+#define ASN1_ERROR -1
+#define ASN1_COMPL_ERROR 1
+#define ASN1_MEMORY_ERROR 0
+#define ASN1_DECODE_ERROR 2
+#define ASN1_TAG_ERROR -3
+#define ASN1_LEN_ERROR -4
+#define ASN1_INDEF_LEN_ERROR -5
+#define ASN1_VALUE_ERROR -6
+
+#define ASN1_CLASS 0xc0
+#define ASN1_FORM 0x20
+#define ASN1_CLASSFORM (ASN1_CLASS | ASN1_FORM)
+#define ASN1_TAG 0x1f
+#define ASN1_LONG_TAG 0x7f
+
+#define ASN1_INDEFINITE_LENGTH 0x80
+#define ASN1_SHORT_DEFINITE_LENGTH 0
+
+#define ASN1_PRIMITIVE 0
+#define ASN1_CONSTRUCTED 0x20
+
+#define ASN1_COMPLETE 1
+#define ASN1_BER_TLV_DECODE 2
+#define ASN1_BER_TLV_PARTIAL_DECODE 3
+
+#define ASN1_NOVALUE 0
+
+#define ASN1_SKIPPED 0
+#define ASN1_OPTIONAL 1
+#define ASN1_CHOOSEN 2
+
+#define CEIL(X,Y) ((X-1) / Y + 1)
+
+#define INVMASK(X,M) (X & (M ^ 0xff))
+#define MASK(X,M) (X & M)
+
+int complete(ErlNifBinary *, unsigned char *, int );
+
+int insert_octets(int, unsigned char **, unsigned char **, int *);
+
+int insert_octets_except_unused(int, unsigned char **, unsigned char **, int *,
+ int);
+
+int insert_octets_as_bits_exact_len(int, int, unsigned char **,
+ unsigned char **, int *);
+
+int insert_octets_as_bits(int, unsigned char **, unsigned char **, int *);
+
+int pad_bits(int, unsigned char **, int *);
+
+int insert_least_sign_bits(int, unsigned char, unsigned char **, int *);
+
+int insert_most_sign_bits(int, unsigned char, unsigned char **, int *);
+
+int insert_bits_as_bits(int, int, unsigned char **, unsigned char **, int *);
+
+int insert_octets_unaligned(int, unsigned char **, unsigned char **, int);
+
+int realloc_memory(ErlNifBinary *, int, unsigned char **);
+
+int decode_begin(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int,
+ unsigned int *);
+
+int decode(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int *, int);
+
+int decode_tag(ErlNifEnv *, ERL_NIF_TERM *, unsigned char *, int, int *);
+
+int decode_value(ErlNifEnv*, ERL_NIF_TERM *, unsigned char *, int *, int, int);
+
+/*
+ *
+ * This section defines functionality for the complete encode of a
+ * PER encoded message
+ *
+ */
+
+int complete(ErlNifBinary *out_binary, unsigned char *in_buf, int in_buf_len) {
+ int counter = in_buf_len;
+ /* counter keeps track of number of bytes left in the
+ input buffer */
+
+ int buf_space = in_buf_len;
+ /* This is the amount of allocated space left of the out_binary. It
+ is possible when padding is applied that more space is needed than
+ was originally allocated. */
+
+ int buf_size = in_buf_len;
+ /* Size of the buffer. May become reallocated and thus other than
+ in_buf_len */
+
+ unsigned char *in_ptr, *ptr;
+ /* in_ptr points at the next byte in in_buf to be moved to
+ complete_buf.
+ ptr points into the new completed buffer, complete_buf, at the
+ position of the next byte that will be set */
+ int unused = 8;
+ /* unused = [1,...,8] indicates how many of the rigthmost bits of
+ the byte that ptr points at that are unassigned */
+
+ int no_bits, no_bytes, in_unused, desired_len, ret, saved_mem, needed,
+ pad_bits;
+
+ unsigned char val;
+
+ in_ptr = in_buf;
+ ptr = out_binary->data;
+ *ptr = 0x00;
+ while (counter > 0) {
+ counter--;
+ switch (*in_ptr) {
+ case 0:
+ /* just one zero-bit should be added to the buffer */
+ if (unused == 1) {
+ unused = 8;
+ *++ptr = 0x00;
+ buf_space--;
+ } else
+ unused--;
+ break;
+
+ case 1:
+ /* one one-bit should be added to the buffer */
+ if (unused == 1) {
+ *ptr = *ptr | 1;
+ unused = 8;
+ *++ptr = 0x00;
+ buf_space--;
+ } else {
+ *ptr = *ptr | (1 << (unused - 1));
+ unused--;
+ }
+ break;
+
+ case 2:
+ /* align buffer to end of byte */
+ if (unused != 8) {
+ *++ptr = 0x00;
+ buf_space--;
+ unused = 8;
+ }
+ break;
+
+ case 10:
+ /* next byte in in_buf tells how many bits in the second next
+ byte that will be used */
+ /* The leftmost unused bits in the value byte are supposed to be
+ zero bits */
+ no_bits = (int) *(++in_ptr);
+ val = *(++in_ptr);
+ counter -= 2;
+ if ((ret = insert_least_sign_bits(no_bits, val, &ptr, &unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 20:
+ /* in this case the next value in_ptr points at holds the number
+ of following bytes that holds the value that will be inserted
+ in the completed buffer */
+ no_bytes = (int) *(++in_ptr);
+ counter -= (no_bytes + 1);
+ if ((counter < 0)
+ || (ret = insert_octets(no_bytes, &in_ptr, &ptr, &unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 21:
+ /* in this case the next two bytes in_ptr points at holds the number
+ of following bytes that holds the value that will be inserted
+ in the completed buffer */
+ no_bytes = (int) *(++in_ptr);
+ no_bytes = no_bytes << 8;
+ no_bytes = no_bytes | (int) *(++in_ptr);
+ counter -= (2 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets(no_bytes, &in_ptr, &ptr, &unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 30:
+ /* If we call the following bytes, in the buffer in_ptr points at,
+ By1,By2,Rest then Rest is the value that will be transfered to
+ the completed buffer. By1 tells how many of the rightmost bits in
+ Rest that should not be used. By2 is the length of Rest in bytes.*/
+ in_unused = (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+ counter -= (2 + no_bytes);
+ ret = -4711;
+ if ((counter < 0)
+ || (ret = insert_octets_except_unused(no_bytes, &in_ptr,
+ &ptr, &unused, in_unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 31:
+ /* If we call the following bytes, in the buffer in_ptr points at,
+ By1,By2,By3,Rest then Rest is the value that will be transfered to
+ the completed buffer. By1 tells how many of the rightmost bits in
+ Rest that should not be used. By2 and By3 is the length of
+ Rest in bytes.*/
+ in_unused = (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+ no_bytes = no_bytes << 8;
+ no_bytes = no_bytes | (int) *(++in_ptr);
+ counter -= (3 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets_except_unused(no_bytes, &in_ptr,
+ &ptr, &unused, in_unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 40:
+ /* This case implies that next byte,By1,(..,By1,By2,Bin,...)
+ is the desired length of the completed value, maybe needs
+ padding zero bits or removal of trailing zero bits from Bin.
+ By2 is the length of Bin and Bin is the value that will be
+ put into the completed buffer. Each byte in Bin has the value
+ 1 or 0.*/
+ desired_len = (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+
+ /* This is the algorithm for need of memory reallocation:
+ Only when padding (cases 40 - 43,45 - 47) more memory may be
+ used than allocated. Therefore one has to keep track of how
+ much of the allocated memory that has been saved, i.e. the
+ difference between the number of parsed bytes of the input buffer
+ and the number of used bytes of the output buffer.
+ If saved memory is less than needed for the padding then we
+ need more memory. */
+ saved_mem = buf_space - counter;
+ pad_bits = desired_len - no_bytes - unused;
+ needed = (pad_bits > 0) ? CEIL(pad_bits,8) : 0;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (2 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets_as_bits_exact_len(desired_len,
+ no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 41:
+ /* Same as case 40 apart from By2, the length of Bin, which is in
+ two bytes*/
+ desired_len = (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+ no_bytes = no_bytes << 8;
+ no_bytes = no_bytes | (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (3 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets_as_bits_exact_len(desired_len,
+ no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 42:
+ /* Same as case 40 apart from By1, the desired length, which is in
+ two bytes*/
+ desired_len = (int) *(++in_ptr);
+ desired_len = desired_len << 8;
+ desired_len = desired_len | (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (3 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets_as_bits_exact_len(desired_len,
+ no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 43:
+ /* Same as case 40 apart from By1 and By2, the desired length and
+ the length of Bin, which are in two bytes each. */
+ desired_len = (int) *(++in_ptr);
+ desired_len = desired_len << 8;
+ desired_len = desired_len | (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+ no_bytes = no_bytes << 8;
+ no_bytes = no_bytes | (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (4 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_octets_as_bits_exact_len(desired_len,
+ no_bytes, &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 45:
+ /* This case assumes that the following bytes in the incoming buffer
+ (called By1,By2,Bin) is By1, which is the number of bits (n) that
+ will be inserted in the completed buffer. By2 is the number of
+ bytes in Bin. Each bit in the buffer Bin should be inserted from
+ the leftmost until the nth.*/
+ desired_len = (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (2 + no_bytes);
+
+ if ((counter < 0)
+ || (ret = insert_bits_as_bits(desired_len, no_bytes,
+ &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 46:
+ /* Same as case 45 apart from By1, the desired length, which is
+ in two bytes. */
+ desired_len = (int) *(++in_ptr);
+ desired_len = desired_len << 8;
+ desired_len = desired_len | (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (3 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_bits_as_bits(desired_len, no_bytes,
+ &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ case 47:
+ /* Same as case 45 apart from By1 and By2, the desired length
+ and the length of Bin, which are in two bytes each. */
+ desired_len = (int) *(++in_ptr);
+ desired_len = desired_len << 8;
+ desired_len = desired_len | (int) *(++in_ptr);
+ no_bytes = (int) *(++in_ptr);
+ no_bytes = no_bytes << 8;
+ no_bytes = no_bytes | (int) *(++in_ptr);
+
+ saved_mem = buf_space - counter;
+ needed = CEIL((desired_len-unused),8) - no_bytes;
+ if (saved_mem < needed) {
+ /* Have to allocate more memory */
+ buf_size += needed;
+ buf_space += needed;
+ if (realloc_memory(out_binary, buf_size, &ptr)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ }
+
+ counter -= (4 + no_bytes);
+ if ((counter < 0)
+ || (ret = insert_bits_as_bits(desired_len, no_bytes,
+ &in_ptr, &ptr, &unused)) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ buf_space -= ret;
+ break;
+
+ default:
+ return ASN1_ERROR;
+ }
+ in_ptr++;
+ }
+ /* The returned buffer must be at least one byte and
+ it must be octet aligned */
+ if ((unused == 8) && (ptr != out_binary->data))
+ return (ptr - out_binary->data);
+ else {
+ ptr++; /* octet align buffer */
+ return (ptr - out_binary->data);
+ }
+}
+
+int realloc_memory(ErlNifBinary *binary, int amount, unsigned char **ptr) {
+
+ int i = *ptr - binary->data;
+
+ if (!enif_realloc_binary(binary, amount)) {
+ /*error handling due to memory allocation failure */
+ return ASN1_ERROR;
+ } else {
+ *ptr = binary->data + i;
+ }
+ return ASN1_OK;
+}
+
+int insert_most_sign_bits(int no_bits, unsigned char val,
+ unsigned char **output_ptr, int *unused) {
+ unsigned char *ptr = *output_ptr;
+
+ if (no_bits < *unused) {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *unused -= no_bits;
+ } else if (no_bits == *unused) {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *unused = 8;
+ *++ptr = 0x00;
+ } else {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *++ptr = 0x00;
+ *ptr = *ptr | (val << *unused);
+ *unused = 8 - (no_bits - *unused);
+ }
+ *output_ptr = ptr;
+ return ASN1_OK;
+}
+
+int insert_least_sign_bits(int no_bits, unsigned char val,
+ unsigned char **output_ptr, int *unused) {
+ unsigned char *ptr = *output_ptr;
+ int ret = 0;
+
+ if (no_bits < *unused) {
+ *ptr = *ptr | (val << (*unused - no_bits));
+ *unused -= no_bits;
+ } else if (no_bits == *unused) {
+ *ptr = *ptr | val;
+ *unused = 8;
+ *++ptr = 0x00;
+ ret++;
+ } else {
+ /* first in the begun byte in the completed buffer insert
+ so many bits that fit, then insert the rest in next byte.*/
+ *ptr = *ptr | (val >> (no_bits - *unused));
+ *++ptr = 0x00;
+ ret++;
+ *ptr = *ptr | (val << (8 - (no_bits - *unused)));
+ *unused = 8 - (no_bits - *unused);
+ }
+ *output_ptr = ptr;
+ return ret;
+}
+
+/* pad_bits adds no_bits bits in the buffer that output_ptr
+ points at.
+ */
+int pad_bits(int no_bits, unsigned char **output_ptr, int *unused) {
+ unsigned char *ptr = *output_ptr;
+ int ret = 0;
+
+ while (no_bits > 0) {
+ if (*unused == 1) {
+ *unused = 8;
+ *++ptr = 0x00;
+ ret++;
+ } else
+ (*unused)--;
+ no_bits--;
+ }
+ *output_ptr = ptr;
+ return ret;
+}
+
+/* insert_bits_as_bits removes no_bytes bytes from the buffer that in_ptr
+ points at and takes the desired_no leftmost bits from those removed
+ bytes and inserts them in the buffer(output buffer) that ptr points at.
+ The unused parameter tells how many bits that are not set in the
+ actual byte in the output buffer. If desired_no is more bits than the
+ input buffer has in no_bytes bytes, then zero bits is padded.*/
+int insert_bits_as_bits(int desired_no, int no_bytes, unsigned char **input_ptr,
+ unsigned char **output_ptr, int *unused) {
+ unsigned char *in_ptr = *input_ptr;
+ unsigned char val;
+ int no_bits, ret, ret2;
+
+ if (desired_no == (no_bytes * 8)) {
+ if (insert_octets_unaligned(no_bytes, &in_ptr, output_ptr, *unused)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ ret = no_bytes;
+ } else if (desired_no < (no_bytes * 8)) {
+ /* printf("insert_bits_as_bits 1\n\r"); */
+ if (insert_octets_unaligned(desired_no / 8, &in_ptr, output_ptr,
+ *unused) == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ /* printf("insert_bits_as_bits 2\n\r"); */
+ val = *++in_ptr;
+ /* printf("val = %d\n\r",(int)val); */
+ no_bits = desired_no % 8;
+ /* printf("no_bits = %d\n\r",no_bits); */
+ insert_most_sign_bits(no_bits, val, output_ptr, unused);
+ ret = CEIL(desired_no,8);
+ } else {
+ if (insert_octets_unaligned(no_bytes, &in_ptr, output_ptr, *unused)
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ ret2 = pad_bits(desired_no - (no_bytes * 8), output_ptr, unused);
+ /* printf("ret2 = %d\n\r",ret2); */
+ ret = CEIL(desired_no,8);
+ /* printf("ret = %d\n\r",ret); */
+ }
+ /* printf("*unused = %d\n\r",*unused); */
+ *input_ptr = in_ptr;
+ return ret;
+}
+
+/* insert_octets_as_bits_exact_len */
+int insert_octets_as_bits_exact_len(int desired_len, int in_buff_len,
+ unsigned char **in_ptr, unsigned char **ptr, int *unused) {
+ int ret = 0;
+ int ret2 = 0;
+
+ if (desired_len == in_buff_len) {
+ if ((ret = insert_octets_as_bits(in_buff_len, in_ptr, ptr, unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ } else if (desired_len > in_buff_len) {
+ if ((ret = insert_octets_as_bits(in_buff_len, in_ptr, ptr, unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ /* now pad with zero bits */
+ /* printf("~npad_bits: called with %d bits padding~n~n~r",desired_len - in_buff_len); */
+ if ((ret2 = pad_bits(desired_len - in_buff_len, ptr, unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ } else {/* desired_len < no_bits */
+ if ((ret = insert_octets_as_bits(desired_len, in_ptr, ptr, unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ /* now remove no_bits - desired_len bytes from in buffer */
+ *in_ptr += (in_buff_len - desired_len);
+ }
+ return (ret + ret2);
+}
+
+/* insert_octets_as_bits takes no_bytes bytes from the buffer that input_ptr
+ points at and inserts the least significant bit of it in the buffer that
+ output_ptr points at. Each byte in the input buffer must be 1 or 0
+ otherwise the function returns ASN1_ERROR. The output buffer is concatenated
+ without alignment.
+ */
+int insert_octets_as_bits(int no_bytes, unsigned char **input_ptr,
+ unsigned char **output_ptr, int *unused) {
+ unsigned char *in_ptr = *input_ptr;
+ unsigned char *ptr = *output_ptr;
+ int used_bits = 8 - *unused;
+
+ while (no_bytes > 0) {
+ switch (*++in_ptr) {
+ case 0:
+ if (*unused == 1) {
+ *unused = 8;
+ *++ptr = 0x00;
+ } else
+ (*unused)--;
+ break;
+ case 1:
+ if (*unused == 1) {
+ *ptr = *ptr | 1;
+ *unused = 8;
+ *++ptr = 0x00;
+ } else {
+ *ptr = *ptr | (1 << (*unused - 1));
+ (*unused)--;
+ }
+ break;
+ default:
+ return ASN1_ERROR;
+ }
+ no_bytes--;
+ }
+ *input_ptr = in_ptr;
+ *output_ptr = ptr;
+ return ((used_bits + no_bytes) / 8); /*return number of new bytes
+ in completed buffer */
+}
+
+/* insert_octets inserts bytes from the input buffer, *input_ptr,
+ into the output buffer, *output_ptr. Before the first byte is
+ inserted the input buffer is aligned.
+ */
+int insert_octets(int no_bytes, unsigned char **input_ptr,
+ unsigned char **output_ptr, int *unused) {
+ unsigned char *in_ptr = *input_ptr;
+ unsigned char *ptr = *output_ptr;
+ int ret = 0;
+
+ if (*unused != 8) {/* must align before octets are added */
+ *++ptr = 0x00;
+ ret++;
+ *unused = 8;
+ }
+ while (no_bytes > 0) {
+ *ptr = *(++in_ptr);
+ *++ptr = 0x00;
+ /* *unused = *unused - 1; */
+ no_bytes--;
+ }
+ *input_ptr = in_ptr;
+ *output_ptr = ptr;
+ return (ret + no_bytes);
+}
+
+/* insert_octets_unaligned inserts bytes from the input buffer, *input_ptr,
+ into the output buffer, *output_ptr.No alignment is done.
+ */
+int insert_octets_unaligned(int no_bytes, unsigned char **input_ptr,
+ unsigned char **output_ptr, int unused) {
+ unsigned char *in_ptr = *input_ptr;
+ unsigned char *ptr = *output_ptr;
+ int n = no_bytes;
+ unsigned char val;
+
+ while (n > 0) {
+ if (unused == 8) {
+ *ptr = *++in_ptr;
+ *++ptr = 0x00;
+ } else {
+ val = *++in_ptr;
+ *ptr = *ptr | val >> (8 - unused);
+ *++ptr = 0x00;
+ *ptr = val << unused;
+ }
+ n--;
+ }
+ *input_ptr = in_ptr;
+ *output_ptr = ptr;
+ return no_bytes;
+}
+
+int insert_octets_except_unused(int no_bytes, unsigned char **input_ptr,
+ unsigned char **output_ptr, int *unused, int in_unused) {
+ unsigned char *in_ptr = *input_ptr;
+ unsigned char *ptr = *output_ptr;
+ int val, no_bits;
+ int ret = 0;
+
+ if (in_unused == 0) {
+ if ((ret = insert_octets_unaligned(no_bytes, &in_ptr, &ptr, *unused))
+ == ASN1_ERROR
+ )
+ return ASN1_ERROR;
+ } else {
+ if ((ret = insert_octets_unaligned(no_bytes - 1, &in_ptr, &ptr, *unused))
+ != ASN1_ERROR) {
+ val = (int) *(++in_ptr);
+ no_bits = 8 - in_unused;
+ /* no_bits is always less than *unused since the buffer is
+ octet aligned after insert:octets call, so the following
+ if clasuse is obsolete I think */
+ if (no_bits < *unused) {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *unused = *unused - no_bits;
+ } else if (no_bits == *unused) {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *++ptr = 0x00;
+ ret++;
+ *unused = 8;
+ } else {
+ *ptr = *ptr | (val >> (8 - *unused));
+ *++ptr = 0x00;
+ ret++;
+ *ptr = *ptr | (val << *unused);
+ *unused = 8 - (no_bits - *unused);
+ }
+ } else
+ return ASN1_ERROR;
+ }
+ *input_ptr = in_ptr;
+ *output_ptr = ptr;
+ return ret;
+}
+
+/*
+ *
+ * This section defines functionality for the partial decode of a
+ * BER encoded message
+ *
+ */
+
+/*
+ * int decode(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf,
+ int in_buf_len, unsigned int *err_pos)
+ * term is a pointer to the term which is to be returned to erlang
+ * in_buf is a pointer into the buffer of incoming bytes.
+ * in_buf_len is the length of the incoming buffer.
+ * The function reads the bytes in the incoming buffer and structures
+ * it in a nested way as Erlang terms. The buffer contains data in the
+ * order tag - length - value. Tag, length and value has the following
+ * format:
+ * A tag is normally one byte but may be of any length, if the tag number
+ * is greater than 30. +----------+
+ * |CL|C|NNNNN|
+ * +----------+
+ * If NNNNN is 31 then will the 7 l.s.b of each of the following tag number
+ * bytes contain the tag number. Each tag number byte that is not the last one
+ * has the m.s.b. set to 1.
+ * The length can be short definite length (sdl), long definite length (ldl)
+ * or indefinite length (il).
+ * sdl: +---------+ the L bits is the length
+ * |0|LLLLLLL|
+ * +---------+
+ * ldl: +---------+ +---------+ +---------+ +-----------+
+ * |1|lllllll| |first len| | | |the Nth len|
+ * +---------+ +---------+ +---------+ ... +-----------+
+ * The first byte tells how many len octets will follow, max 127
+ * il: +---------+ +----------------------+ +--------+ +--------+
+ * |1|0000000| |content octets (Value)| |00000000| |00000000|
+ * +---------+ +----------------------+ +--------+ +--------+
+ * The value octets are preceded by one octet and followed by two
+ * exactly as above. The value must be some tag-length-value encoding.
+ *
+ * The function returns a value in Erlang nif term format:
+ * {{TagNo,Value},Rest}
+ * TagNo is an integer ((CL bsl 16) + tag number) which limits the tag number
+ * to 65535.
+ * Value is a binary if the C bit in tag was unset, otherwise (if tag was
+ * constructed) Value is a list, List.
+ * List is like: [{TagNo,Value},{TagNo,Value},...]
+ * Rest is a binary, i.e. the undecoded part of the buffer. Most often Rest
+ * is the empty binary.
+ * If some error occured during the decoding of the in_buf an error is returned.
+ */
+int decode_begin(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf,
+ int in_buf_len, unsigned int *err_pos) {
+ int maybe_ret;
+ int ib_index = 0;
+ unsigned char *rest_data;
+ ERL_NIF_TERM decoded_term, rest;
+
+ if ((maybe_ret = decode(env, &decoded_term, in_buf, &ib_index, in_buf_len))
+ <= ASN1_ERROR)
+ {
+ *err_pos = ib_index;
+ return maybe_ret;
+ };
+
+ // The remaining binary after one ASN1 segment has been decoded
+ if ((rest_data = enif_make_new_binary(env, in_buf_len - ib_index, &rest))
+ == NULL) {
+ *term = enif_make_atom(env, "could_not_alloc_binary");
+ return ASN1_ERROR;
+ }
+
+ *term = enif_make_tuple2(env, decoded_term, rest);
+ return ASN1_OK;
+}
+
+int decode(ErlNifEnv* env, ERL_NIF_TERM *term, unsigned char *in_buf,
+ int *ib_index, int in_buf_len) {
+ int maybe_ret;
+ int form;
+ ERL_NIF_TERM tag, value;
+
+ /*buffer must hold at least two bytes*/
+ if ((*ib_index + 2) > in_buf_len)
+ return ASN1_VALUE_ERROR;
+ /* "{{TagNo," */
+ if ((form = decode_tag(env, &tag, in_buf, in_buf_len, ib_index))
+ <= ASN1_ERROR
+ )
+ return form; /* 5 bytes */
+ if (*ib_index >= in_buf_len) {
+ return ASN1_TAG_ERROR;
+ }
+ /* buffer must hold at least one byte (0 as length and nothing as
+ value) */
+ /* "{{TagNo,Value}," */
+ if ((maybe_ret = decode_value(env, &value, in_buf, ib_index, form,
+ in_buf_len)) <= ASN1_ERROR
+ )
+ return maybe_ret; /* at least 5 bytes */
+ *term = enif_make_tuple2(env, tag, value);
+ return ASN1_OK;
+}
+
+/*
+ * decode_tag decodes the BER encoded tag in in_buf and creates an
+ * nif term tag
+ */
+int decode_tag(ErlNifEnv* env, ERL_NIF_TERM *tag, unsigned char *in_buf,
+ int in_buf_len, int *ib_index) {
+ int tag_no, tmp_tag, form;
+
+ /* first get the class of tag and bit shift left 16*/
+ tag_no = ((MASK(in_buf[*ib_index],ASN1_CLASS)) << 10);
+
+ form = (MASK(in_buf[*ib_index],ASN1_FORM));
+
+ /* then get the tag number */
+ if ((tmp_tag = (int) INVMASK(in_buf[*ib_index],ASN1_CLASSFORM)) < 31) {
+ *tag = enif_make_uint(env, tag_no + tmp_tag);
+ (*ib_index)++;
+ } else {
+ int n = 0; /* n is used to check that the 64K limit is not
+ exceeded*/
+
+ /* should check that at least three bytes are left in
+ in-buffer,at least two tag byte and at least one length byte */
+ if ((*ib_index + 3) > in_buf_len)
+ return ASN1_VALUE_ERROR;
+ (*ib_index)++;
+ /* The tag is in the following bytes in in_buf as
+ 1ttttttt 1ttttttt ... 0ttttttt, where the t-bits
+ is the tag number*/
+ /* In practice is the tag size limited to 64K, i.e. 16 bits. If
+ the tag is greater then 64K return an error */
+ while (((tmp_tag = (int) in_buf[*ib_index]) >= 128) && n < 2) {
+ /* m.s.b. = 1 */
+ tag_no = tag_no + (MASK(tmp_tag,ASN1_LONG_TAG) << 7);
+ (*ib_index)++;
+ n++;
+ };
+ if ((n == 2) && in_buf[*ib_index] > 3)
+ return ASN1_TAG_ERROR; /* tag number > 64K */
+ tag_no = tag_no + in_buf[*ib_index];
+ (*ib_index)++;
+ *tag = enif_make_uint(env, tag_no);
+ }
+ return form;
+}
+
+/*
+ * decode_value decodes the BER encoded length and value fields in the
+ * in_buf and puts the value part in the decode_buf as an Erlang
+ * nif term into value
+ */
+int decode_value(ErlNifEnv* env, ERL_NIF_TERM *value, unsigned char *in_buf,
+ int *ib_index, int form, int in_buf_len) {
+ int maybe_ret;
+ unsigned int len = 0;
+ unsigned int lenoflen = 0;
+ int indef = 0;
+ unsigned char *tmp_out_buff;
+ ERL_NIF_TERM term = 0, curr_head = 0;
+
+ if (((in_buf[*ib_index]) & 0x80) == ASN1_SHORT_DEFINITE_LENGTH) {
+ len = in_buf[*ib_index];
+ } else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH
+ )
+ indef = 1;
+ else /* long definite length */{
+ lenoflen = (in_buf[*ib_index] & 0x7f); /*length of length */
+ if (lenoflen > (in_buf_len - (*ib_index + 1)))
+ return ASN1_LEN_ERROR;
+ len = 0;
+ while (lenoflen--) {
+ (*ib_index)++;
+ if (!(len < (1 << (sizeof(len) - 1) * 8)))
+ return ASN1_LEN_ERROR; /* length does not fit in 32 bits */
+ len = (len << 8) + in_buf[*ib_index];
+ }
+ }
+ if (len > (in_buf_len - (*ib_index + 1)))
+ return ASN1_VALUE_ERROR;
+ (*ib_index)++;
+ if (indef == 1) { /* in this case it is desireably to check that indefinite length
+ end bytes exist in inbuffer */
+ curr_head = enif_make_list(env, 0);
+ while (!(in_buf[*ib_index] == 0 && in_buf[*ib_index + 1] == 0)) {
+ if (*ib_index >= in_buf_len)
+ return ASN1_INDEF_LEN_ERROR;
+
+ if ((maybe_ret = decode(env, &term, in_buf, ib_index, in_buf_len))
+ <= ASN1_ERROR
+ )
+ return maybe_ret;
+ curr_head = enif_make_list_cell(env, term, curr_head);
+ }
+ enif_make_reverse_list(env, curr_head, value);
+ (*ib_index) += 2; /* skip the indefinite length end bytes */
+ } else if (form == ASN1_CONSTRUCTED)
+ {
+ int end_index = *ib_index + len;
+ if (end_index > in_buf_len)
+ return ASN1_LEN_ERROR;
+ curr_head = enif_make_list(env, 0);
+ while (*ib_index < end_index) {
+
+ if ((maybe_ret = decode(env, &term, in_buf, ib_index, in_buf_len))
+ <= ASN1_ERROR
+ )
+ return maybe_ret;
+ curr_head = enif_make_list_cell(env, term, curr_head);
+ }
+ enif_make_reverse_list(env, curr_head, value);
+ } else {
+ if ((*ib_index + len) > in_buf_len)
+ return ASN1_LEN_ERROR;
+ tmp_out_buff = enif_make_new_binary(env, len, value);
+ memcpy(tmp_out_buff, in_buf + *ib_index, len);
+ *ib_index = *ib_index + len;
+ }
+ return ASN1_OK;
+}
+
+static ERL_NIF_TERM encode_per_complete(ErlNifEnv* env, int argc,
+ const ERL_NIF_TERM argv[]) {
+ ERL_NIF_TERM err_code;
+ ErlNifBinary in_binary;
+ ErlNifBinary out_binary;
+ int complete_len;
+ if (!enif_inspect_iolist_as_binary(env, argv[0], &in_binary))
+ return enif_make_atom(env, "badarg");
+
+ if (!enif_alloc_binary(in_binary.size, &out_binary))
+ return enif_make_atom(env, "alloc_binary_failed");
+
+ if (in_binary.size == 0)
+ return enif_make_binary(env, &out_binary);
+
+ if ((complete_len = complete(&out_binary, in_binary.data, in_binary.size))
+ <= ASN1_ERROR) {
+ enif_release_binary(&out_binary);
+ if (complete_len == ASN1_ERROR
+ )
+ err_code = enif_make_uint(env, '1');
+ else
+ err_code = enif_make_uint(env, 0);
+ return enif_make_tuple2(env, enif_make_atom(env, "error"), err_code);
+ }
+ if (complete_len < out_binary.size)
+ enif_realloc_binary(&out_binary, complete_len);
+
+ return enif_make_binary(env, &out_binary);
+}
+
+static ERL_NIF_TERM decode_ber_tlv(ErlNifEnv* env, int argc,
+ const ERL_NIF_TERM argv[]) {
+ ErlNifBinary in_binary;
+ ERL_NIF_TERM return_term;
+ unsigned int err_pos = 0, return_code;
+
+ if (!enif_inspect_iolist_as_binary(env, argv[0], &in_binary))
+ return enif_make_badarg(env);
+
+ if ((return_code = decode_begin(env, &return_term, in_binary.data,
+ in_binary.size, &err_pos)) != ASN1_OK
+ )
+ return enif_make_tuple2(env, enif_make_atom(env,"error"), enif_make_tuple2(env,
+ enif_make_int(env, return_code),enif_make_int(env, err_pos)));
+ return return_term;
+}
+
+static int is_ok_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) {
+ int i;
+ return enif_get_int(env, load_info, &i) && i == 1;
+}
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) {
+ if (!is_ok_load_info(env, load_info))
+ return -1;
+ return 0;
+}
+
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data,
+ ERL_NIF_TERM load_info) {
+ if (!is_ok_load_info(env, load_info))
+ return -1;
+ return 0;
+}
+
+static void unload(ErlNifEnv* env, void* priv_data) {
+
+}
+
+static ErlNifFunc nif_funcs[] = { { "encode_per_complete", 1,
+ encode_per_complete }, { "decode_ber_tlv", 1, decode_ber_tlv } };
+
+ERL_NIF_INIT(asn1rt_nif, nif_funcs, load, NULL, upgrade, unload)
diff --git a/lib/asn1/doc/src/asn1_spec.xmlsrc b/lib/asn1/doc/src/asn1_spec.xmlsrc
index 8d61834da8..07cba17816 100644
--- a/lib/asn1/doc/src/asn1_spec.xmlsrc
+++ b/lib/asn1/doc/src/asn1_spec.xmlsrc
@@ -4,7 +4,7 @@
<chapter>
<header>
<copyright>
- <year>2003</year><year>2009</year>
+ <year>2003</year><year>2011</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -43,7 +43,7 @@
<p>So far this functionality is only provided when using the
optimized BER_BIN version, that is when compiling with the
options <c>ber_bin</c> and <c>optimize</c>. It does also work
- using the <c>driver</c> option. We have no intent to make this
+ using the <c>nif</c> option. We have no intent to make this
available on the default BER version, but maybe in the PER_BIN
version (<c>per_bin</c>).
</p>
@@ -661,7 +661,9 @@ ValAction = {'Action',17,{'Button',4711,false}}.
<p>The ASN.1 specs in the test are compiled with the options
<c>ber_bin, optimize, driver</c> and <c>asn1config</c>. If the
<c>driver</c> option had been omitted there should have been
- higher values for <c>decode</c> and <c>decode_part</c>.
+ higher values for <c>decode</c> and <c>decode_part</c>. These tests have
+ not been re-run using nifs, but are expected to perform about 5% better
+ than the linked-in driver.
</p>
<p>The test program runs 10000 decodes on the value, resulting
in a printout with the elapsed time in microseconds for the
diff --git a/lib/asn1/doc/src/asn1_ug.xml b/lib/asn1/doc/src/asn1_ug.xml
index 12d986308f..61e4f6062f 100644
--- a/lib/asn1/doc/src/asn1_ug.xml
+++ b/lib/asn1/doc/src/asn1_ug.xml
@@ -4,7 +4,7 @@
<chapter>
<header>
<copyright>
- <year>1997</year><year>2010</year>
+ <year>1997</year><year>2011</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -347,7 +347,7 @@ erlc -o ../asnfiles -I ../asnfiles -I /usr/local/standards/asn1 Person.asn
<p>This flag has effect only when used together with one of
<c>per_bin</c> or <c>ber_bin</c> flags. It gives time optimized
code in the generated modules and it uses another runtime module.
- In the <c>per_bin</c> case a linked-in driver is used. The
+ In the <c>per_bin</c> case a nif is used. The
result from an encode is a binary.</p>
<p><em>When this flag is used you cannot use the old format</em><c>{TypeName,Value}</c> when you encode values. Since it is
an unnecessary construct it has been removed in favor of
@@ -362,9 +362,14 @@ erlc -o ../asnfiles -I ../asnfiles -I /usr/local/standards/asn1 Person.asn
</item>
<tag><c>+driver</c></tag>
<item>
- <p>Together with the flags <c>ber_bin</c> and <c>optimize</c>
- you choose to use a linked in driver for considerable faster
- decode.</p>
+ <p>As of R15B this means the same as the <c>nif</c> option. Kept for
+ backwards compatability reasons.</p>
+ </item>
+ <tag><c>+nif</c></tag>
+ <item>
+ <p>Together with the flags <c>ber_bin</c>
+ and <c>optimize</c> you choose to use a nif for considerable
+ faster decode. </p>
</item>
<tag><c>+asn1config</c></tag>
<item>
@@ -492,7 +497,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
</row>
<row>
<cell align="left" valign="middle">BER</cell>
- <cell align="left" valign="middle"><em>[ber_bin, optimize, driver]</em></cell>
+ <cell align="left" valign="middle"><em>[ber_bin, optimize, nif]</em></cell>
<cell align="left" valign="middle">EAVF</cell>
<cell align="left" valign="middle">iolist</cell>
<cell align="left" valign="middle">iolist / binary</cell>
@@ -557,7 +562,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
</row>
<row>
<cell align="left" valign="middle">DER</cell>
- <cell align="left" valign="middle"><em>[ber_bin, optimize, driver, der]</em></cell>
+ <cell align="left" valign="middle"><em>[ber_bin, optimize, nif, der]</em></cell>
<cell align="left" valign="middle">EAVF</cell>
<cell align="left" valign="middle">iolist</cell>
<cell align="left" valign="middle">binary</cell>
@@ -626,23 +631,24 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
</table>
<p>
- The sole compile options <c>ber</c>, <c>ber_bin</c> and <c>per</c>
- are kept for backwards compatibility and should not be used in
- new code.
+ The sole compile options <c>ber</c>, <c>ber_bin</c>, <c>per</c> and
+ <c>driver</c> are kept for backwards compatibility and should not be
+ used in new code. The nif implementation which replaces the linked-in
+ driver has been shown to be about 5-15% faster.
</p>
<p>
You are strongly recommended to use the appropriate alternative
of the bold typed options. The <c>optimize</c> and
- <c>driver</c> options does not affect the encode or decode
+ <c>nif</c> options does not affect the encode or decode
result, just the time spent in run-time. When <c>ber_bin</c> and
- <c>driver</c> or <c>per_bin, optimize</c> and <c>driver</c> is
- combined the C-code driver is used in chosen parts of encode /
+ <c>nif</c> or <c>per_bin</c> and <c>optimize</c> is
+ combined the C-code nif is used in chosen parts of encode /
decode procedure.
</p>
<table>
<row>
<cell align="left" valign="middle"><em>Compile options, allowed combinations</em></cell>
- <cell align="left" valign="middle"><em>use of linked-in driver</em></cell>
+ <cell align="left" valign="middle"><em>use of nif</em></cell>
</row>
<row>
<cell align="left" valign="middle">[ber]</cell>
@@ -657,7 +663,7 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
<cell align="left" valign="middle">no</cell>
</row>
<row>
- <cell align="left" valign="middle"><em>[ber_bin, optimize, driver]</em></cell>
+ <cell align="left" valign="middle"><em>[ber_bin, optimize, nif]</em></cell>
<cell align="left" valign="middle">yes</cell>
</row>
<row>
@@ -690,12 +696,12 @@ asn1ct:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
<cell align="left" valign="middle">no</cell>
</row>
<row>
- <cell align="left" valign="middle"><em>[ber_bin, optimize, driver, der]</em></cell>
+ <cell align="left" valign="middle"><em>[ber_bin, optimize, nif, der]</em></cell>
<cell align="left" valign="middle">yes</cell>
</row>
- <tcaption>When the ASN1 linked-in driver is used.</tcaption>
+ <tcaption>When the ASN1 nif is used.</tcaption>
</table>
</section>
@@ -712,14 +718,14 @@ asn1rt:decode('H323-MESSAGES','SomeChoiceType',Bytes). </pre>
<pre>
'H323-MESSAGES':encode('SomeChoiceType',{call,"octetstring"}).
'H323-MESSAGES':decode('SomeChoiceType',Bytes). </pre>
- <p>The asn1 linked-in driver is enabled in two occasions: encoding of
+ <p>The asn1 nif is enabled in two occasions: encoding of
asn1 values when the asn1 spec is compiled with <c>per_bin</c> and
<c>optimize</c> or decode of encoded asn1 values when the asn1 spec is
- compiled with <c>ber_bin</c>, <c>optimize</c> and <c>driver</c>. In
- those cases the driver will be loaded automatically at the first call
+ compiled with <c>ber_bin</c>, <c>optimize</c> and <c>nif</c>. In
+ those cases the nif will be loaded automatically at the first call
to <c>encode</c>/<c>decode</c>. If one doesn't want the performance
- overhead of the driver being loaded at the first call it is possible
- to load the driver separately by <c>asn1rt:load_driver()</c>. </p>
+ overhead of the nif being loaded at the first call it is possible
+ to load the nif separately by loading the <c>asn1rt_nif</c> module.</p>
<p>By invoking the function <c>info/0</c> in a generated module, one
gets information about which compiler options were used.</p>
</section>
diff --git a/lib/asn1/doc/src/asn1ct.xml b/lib/asn1/doc/src/asn1ct.xml
index 265f8735c2..d7c2572dc8 100644
--- a/lib/asn1/doc/src/asn1ct.xml
+++ b/lib/asn1/doc/src/asn1ct.xml
@@ -4,7 +4,7 @@
<erlref>
<header>
<copyright>
- <year>1997</year><year>2010</year>
+ <year>1997</year><year>2011</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -52,7 +52,7 @@
<v>Options = [Option| OldOption]</v>
<v>Option = ber_bin | per_bin | uper_bin | der | compact_bit_string |
noobj | {n2n,EnumTypeName} |{outdir,Dir} | {i,IncludeDir} | optimize |
- driver | asn1config | undec_rest | {inline,OutputName} | inline |
+ nif | asn1config | undec_rest | {inline,OutputName} | inline |
{macro_name_prefix, Prefix} | {record_name_prefix, Prefix} | verbose</v>
<v>OldOption = ber | per</v>
<v>Reason = term()</v>
@@ -212,16 +212,21 @@ Binary = binary()
<c>per_bin</c>
or <c>ber_bin</c> option. It gives time optimized code
generated and it uses another runtime module and
- in the <c>per_bin</c> case a linked-in driver. The result
+ in the <c>per_bin</c> case a nif. The result
in the <c>per_bin</c> case from an encode when compiled
with this option will be a binary.</p>
</item>
<tag><c>driver</c></tag>
<item>
+ <p>As of R15B this means the same as the <c>nif</c> option. Kept for
+ backwards compatability reasons.</p>
+ </item>
+ <tag><c>nif</c></tag>
+ <item>
<p>Option valid together with <c>ber_bin</c> and <c>optimize</c>
- options. It enables the use of a linked-in driver that gives
- considerable faster decode. In <c>ber_bin</c> the driver is
- enabled only by explicit use of the option <c>driver</c>.</p>
+ options. It enables the use of several nifs that gives faster
+ decode. Nifs are only enabled by the explicit use of the option
+ <c>nif</c></p>
</item>
<tag><c>asn1config</c></tag>
<item>
@@ -343,18 +348,6 @@ Binary = binary()
</desc>
</func>
<func>
- <name>validate(Module,Type,Value) -> ok | {error,Reason}</name>
- <fsummary>Validate an ASN.1 value.</fsummary>
- <type>
- <v>Module = Type = atom()</v>
- <v>Value = term()</v>
- </type>
- <desc>
- <p>Validates that <c>Value</c> conforms to <c>Type</c>
- from <c>Module</c>. <em>Not implemented in this version of the ASN.1 application.</em></p>
- </desc>
- </func>
- <func>
<name>value(Module ,Type) -> {ok,Value} | {error,Reason}</name>
<fsummary>Create an ASN.1 value for test purposes.</fsummary>
<type>
diff --git a/lib/asn1/doc/src/asn1rt.xml b/lib/asn1/doc/src/asn1rt.xml
index 1217a07e9b..0c3c257189 100644
--- a/lib/asn1/doc/src/asn1rt.xml
+++ b/lib/asn1/doc/src/asn1rt.xml
@@ -4,7 +4,7 @@
<erlref>
<header>
<copyright>
- <year>1997</year><year>2009</year>
+ <year>1997</year><year>2011</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -42,36 +42,6 @@
<funcs>
<func>
- <name>start() -> ok |{error,Reason}</name>
- <fsummary>Starts the asn1 server.</fsummary>
- <type>
- <v>Reason = term()</v>
- </type>
- <desc>
- <p>Starts the asn1 server that loads the drivers.</p>
- <p>The server schedules a driver that is not blocked by
- another caller. The driver is used by the asn1 application if
- specs are compiled with options <c>[per_bin, optimize]</c> or
- <c>[ber_bin, optimize, driver]</c>. The server will be started
- automatically at encode/decode if it isn't done explicitly. If
- encode/decode with driver is used in test or industrial code
- it is a performance gain to start it explicitly to avoid the
- one time load in run-time.</p>
- </desc>
- </func>
-
- <func>
- <name>stop() -> ok |{error,Reason}</name>
- <fsummary>Stops the asn1 server.</fsummary>
- <type>
- <v>Reason = term()</v>
- </type>
- <desc>
- <p>Stops the asn1 server and unloads the drivers.</p>
- </desc>
- </func>
-
- <func>
<name>decode(Module,Type,Bytes) -> {ok,Value}|{error,Reason}</name>
<fsummary>Decode from bytes into an ASN.1 value.</fsummary>
<type>
@@ -126,35 +96,23 @@
<func>
<name>load_driver() -> ok | {error,Reason}</name>
- <fsummary>Loads the linked-in driver.</fsummary>
+ <fsummary>Loads the linked-in driver. (deprecated)</fsummary>
<type>
<v>Reason = term()</v>
</type>
<desc>
- <p>This function loads the linked-in driver before the first call
- to encode. If this function is not called the driver will be loaded
- automatically at the first call to encode. If one doesn't want the
- performance cost of a driver load when the application is running,
- this function makes it possible to load the driver in an
- initialization.</p>
- <p>The driver is only used when encoding/decoding ASN.1 files that
- were compiled with the options <c>per_bin</c> and <c>optimize</c>.</p>
+ <p>This function is obsolete and will be removed in R16A</p>
</desc>
</func>
<func>
<name>unload_driver() -> ok | {error,Reason}</name>
- <fsummary>Unloads the linked-in driver.</fsummary>
+ <fsummary>Unloads the linked-in driver. (deprecated)</fsummary>
<type>
<v>Reason = term()</v>
</type>
<desc>
- <p>This function unloads the linked-in driver.
- When the driver has been loaded it remains in the environment until
- it is unloaded. Normally the driver should remain loaded, it is
- crucial for the performance of ASN.1 encoding. </p>
- <p>The driver is only used when ASN.1 modules have been compiled
- with the flags <c>per_bin</c> and <c>optimize</c>.</p>
+ <p>This function is obsolete and will be removed in R16A</p>
</desc>
</func>
@@ -188,19 +146,6 @@
value, to a UTF8 encoded binary.</p>
</desc>
</func>
-
- <func>
- <name>validate(Module,Type,Value) -> ok | {error,Reason}</name>
- <fsummary>Validate an ASN.1 value.</fsummary>
- <type>
- <v>Module = Type = atom()</v>
- <v>Value = term()</v>
- </type>
- <desc>
- <p>Validates that <c>Value</c> conforms to <c>Type</c>
- from <c>Module</c>. <em>Not implemented in this version of the ASN.1 application.</em></p>
- </desc>
- </func>
</funcs>
diff --git a/lib/asn1/src/Makefile b/lib/asn1/src/Makefile
index 2733cde3f8..3a59773d93 100644
--- a/lib/asn1/src/Makefile
+++ b/lib/asn1/src/Makefile
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -68,7 +68,7 @@ RT_MODULES= \
asn1rt_per_bin_rt2ct \
asn1rt_uper_bin \
asn1rt_check \
- asn1rt_driver_handler
+ asn1rt_nif
# asn1_sup \
# asn1_app \
# asn1_server
diff --git a/lib/asn1/src/asn1.app.src b/lib/asn1/src/asn1.app.src
index abacb0a1e9..09144ba2f7 100644
--- a/lib/asn1/src/asn1.app.src
+++ b/lib/asn1/src/asn1.app.src
@@ -9,12 +9,11 @@
asn1rt_ber_bin,
asn1rt_ber_bin_v2,
asn1rt_check,
- asn1rt_driver_handler
+ asn1rt_nif
]},
{registered, [
asn1_ns,
- asn1db,
- asn1_driver_owner
+ asn1db
]},
{env, []},
{applications, [kernel, stdlib]}
diff --git a/lib/asn1/src/asn1_app.erl b/lib/asn1/src/asn1_app.erl
index 2d3eed1743..9fff96e0bf 100644
--- a/lib/asn1/src/asn1_app.erl
+++ b/lib/asn1/src/asn1_app.erl
@@ -28,7 +28,7 @@
%% {error, Reason}
%%
start(_Type, _StartArgs) ->
- asn1_sup:start_link().
+ {ok, self()}.
%% stop(State)
%%
diff --git a/lib/asn1/src/asn1_server.erl b/lib/asn1/src/asn1_server.erl
deleted file mode 100644
index aeb59d8b0c..0000000000
--- a/lib/asn1/src/asn1_server.erl
+++ /dev/null
@@ -1,107 +0,0 @@
-%% ``The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved via the world wide web at http://www.erlang.org/.
-%%
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%%
-%% The Initial Developer of the Original Code is Ericsson Utvecklings AB.
-%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
-%% AB. All Rights Reserved.''
-%%
-%% $Id$
-%%
-
-%% Purpose: Provide complete encode/and pre-decode of asn1.
--module(asn1_server).
-
-
-
--behaviour(gen_server).
-
--export([start_link/0,client_port/0]).
-
-%% Internal exports, call-back functions.
--export([init/1,handle_call/3,handle_cast/2,handle_info/2,code_change/3,
- terminate/2]).
-
-
-%% Macros
--define(port_names,
- { asn1_drv01, asn1_drv02, asn1_drv03, asn1_drv04,
- asn1_drv05, asn1_drv06, asn1_drv07, asn1_drv08,
- asn1_drv09, asn1_drv10, asn1_drv11, asn1_drv12,
- asn1_drv13, asn1_drv14, asn1_drv15, asn1_drv16 }).
-%%% --------------------------------------------------------
-%%% Interface Functions.
-%%% --------------------------------------------------------
-
-start_link() ->
- gen_server:start_link({local, asn1_server}, asn1_server, [], []).
-
-init([]) ->
- process_flag(trap_exit, true),
- erl_ddll:start(),
- PrivDir = code:priv_dir(asn1),
- LibDir1 = filename:join([PrivDir, "lib"]),
- case erl_ddll:load_driver(LibDir1, asn1_erl_drv) of
- ok -> ok;
- {error,_} ->
- LibDir2 =
- filename:join(LibDir1,
- erlang:system_info(system_architecture)),
- erl_ddll:load_driver(LibDir2, asn1_erl_drv)
- end,
- open_ports("asn1_erl_drv",size(?port_names)).
-
-open_ports(_,0) ->
- {ok, []};
-open_ports(Cmd,N) ->
- Port = open_port({spawn, Cmd}, []),
- %% check that driver is loaded, linked and working
- case catch port_control(Port, 0, []) of
- {'EXIT', _} ->
- {stop, nodriver};
- _ ->
- register(element(N,?port_names), Port),
- open_ports(Cmd,N-1)
- end.
-
-client_port() ->
- element(erlang:system_info(scheduler_id) rem size(?port_names) + 1,
- ?port_names).
-
-
-%%% --------------------------------------------------------
-%%% The call-back functions.
-%%% --------------------------------------------------------
-
-handle_call(_, _, State) ->
- {noreply, State}.
-
-handle_cast(_, State) ->
- {noreply, State}.
-
-handle_info({'EXIT', Pid, _Reason}, State) when is_pid(Pid) ->
- {noreply, State};
-
-handle_info({'EXIT', Port, Reason}, State) when is_port(Port) ->
- {stop, {port_died, Reason}, State};
-handle_info(_, State) ->
- {noreply, State}.
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-terminate(_Reason, _State) ->
- close_ports(size(?port_names)).
-
-close_ports(0) ->
- ok;
-close_ports(N) ->
- element(N,?port_names) ! {self(), close}, %% almost same as port_close(Name)
- close_ports(N-1).
diff --git a/lib/asn1/src/asn1ct.erl b/lib/asn1/src/asn1ct.erl
index a167d27f82..a170dd8660 100644
--- a/lib/asn1/src/asn1ct.erl
+++ b/lib/asn1/src/asn1ct.erl
@@ -47,6 +47,10 @@
-import(asn1ct_gen_ber_bin_v2,[encode_tag_val/3,decode_class/1]).
+-ifndef(vsn).
+-define(vsn,"0.0.1").
+-endif.
+
-define(unique_names,0).
-define(dupl_uniquedefs,1).
-define(dupl_equaldefs,2).
@@ -81,6 +85,12 @@ compile(File) ->
compile(File,[]).
compile(File,Options) when is_list(Options) ->
+ case lists:member(driver, Options) of %% remove me in R16A!
+ true ->
+ io:format("Warning: driver option is obsolete and will be removed in R16A, use nif instead!");
+ false ->
+ ok
+ end,
Options1 = optimize_ber_bin(Options),
Options2 = includes(File,Options1),
Includes=[I||{i,I}<-Options2],
@@ -1082,7 +1092,7 @@ get_runtime_mod(Options) ->
ber_bin_v2 -> ["asn1rt_ber_bin_v2.erl"];
uper_bin -> ["asn1rt_uper_bin.erl"]
end,
- RtMod1++["asn1rt_check.erl","asn1rt_driver_handler.erl","asn1rt.erl"].
+ RtMod1++["asn1rt_check.erl","asn1rt.erl"].
erl_compile(OutFile,Options) ->
diff --git a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl
index e3be914af4..e16873c717 100644
--- a/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl
+++ b/lib/asn1/src/asn1ct_constructed_ber_bin_v2.erl
@@ -1227,7 +1227,7 @@ gen_dec_call({typefield,_},_,_,_Cname,Type,BytesVar,Tag,_,_,false,_) ->
emit([nl,indent(6),"begin",nl]),
% emit([indent(9),{curr,opendec}," = ?RT_BER:decode_open_type(",
emit([indent(9),{curr,tmptlv}," = ?RT_BER:decode_open_type(",
- BytesVar,",",{asis,Tag},"),",nl]),
+ BytesVar,",",{asis,Tag},asn1ct_gen:nif_parameter(),"),",nl]),
% emit([indent(9),"{",{curr,tmptlv},",_} = ?RT_BER:decode(",
% {curr,opendec},"),",nl]),
@@ -1242,7 +1242,8 @@ gen_dec_call({typefield,_},_,_,_Cname,Type,BytesVar,Tag,_,_,false,_) ->
emit([indent(9),"end",nl,indent(6),"end",nl]),
[];
gen_dec_call({typefield,_},_,_,Cname,Type,BytesVar,Tag,_,_,_DecObjInf,OptOrMandComp) ->
- emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},")"]),
+ emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},
+ asn1ct_gen:nif_parameter(),")"]),
RefedFieldName =
% asn1ct_gen:get_constraint(Type#type.constraint,
% tableconstraint_info),
@@ -1250,7 +1251,8 @@ gen_dec_call({typefield,_},_,_,Cname,Type,BytesVar,Tag,_,_,_DecObjInf,OptOrMandC
[{Cname,RefedFieldName,asn1ct_gen:mk_var(asn1ct_name:curr(term)),
asn1ct_gen:mk_var(asn1ct_name:curr(tmpterm)),Tag,OptOrMandComp}];
gen_dec_call({objectfield,PrimFieldName,PFNList},_,_,Cname,_,BytesVar,Tag,_,_,_,OptOrMandComp) ->
- emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},")"]),
+ emit(["?RT_BER:decode_open_type(",BytesVar,",",{asis,Tag},
+ asn1ct_gen:nif_parameter(),")"]),
[{Cname,{PrimFieldName,PFNList},asn1ct_gen:mk_var(asn1ct_name:curr(term)),
asn1ct_gen:mk_var(asn1ct_name:curr(tmpterm)),Tag,OptOrMandComp}];
gen_dec_call(InnerType,Erules,TopType,Cname,Type,BytesVar,Tag,PrimOptOrMand,
diff --git a/lib/asn1/src/asn1ct_constructed_per.erl b/lib/asn1/src/asn1ct_constructed_per.erl
index c1b6aa5713..e07680f10b 100644
--- a/lib/asn1/src/asn1ct_constructed_per.erl
+++ b/lib/asn1/src/asn1ct_constructed_per.erl
@@ -73,16 +73,23 @@ gen_encode_constructed(Erule,Typename,D) when is_record(D,type) ->
_ ->
ok
end,
- case {Optionals = optionals(to_textual_order(CompList)),CompList} of
- {[],EmptyCL} when EmptyCL == {[],[],[]};EmptyCL == {[],[]};EmptyCL == [] ->
+ case {Optionals = optionals(to_textual_order(CompList)),CompList,
+ is_optimized(Erule)} of
+ {[],EmptyCL,_} when EmptyCL == {[],[],[]};EmptyCL == {[],[]};EmptyCL == [] ->
emit(["%%Variable setting just to eliminate ",
"compiler warning for unused vars!",nl,
"_Val = ",{curr,val},",",nl]);
- {[],_} ->
+ {[],_,_} ->
emit([{next,val}," = ?RT_PER:list_to_record("]),
emit(["'",asn1ct_gen:list2rname(Typename),"'"]),
emit([", ",{curr,val},"),",nl]);
- _ ->
+ {_,_,true} ->
+ gen_fixoptionals(Optionals),
+ FixOpts = param_map(fun(Var) ->
+ {var,Var}
+ end,asn1ct_name:all(fixopt)),
+ emit({"{",{next,val},",Opt} = {",{curr,val},",[",FixOpts,"]},",nl});
+ {_,_,false} ->
Fixoptcall = ",Opt} = ?RT_PER:fixoptionals(",
emit({"{",{next,val},Fixoptcall,
{asis,Optionals},",",length(Optionals),
@@ -439,9 +446,7 @@ gen_encode_sof(Erule,Typename,SeqOrSetOf,D) when is_record(D,type) ->
_->
""
end,
- emit({nl,indent(3),"?RT_PER:encode_length(",
- {asis,SizeConstraint},
- ",length(Val)),",nl}),
+ gen_encode_length(SizeConstraint, is_optimized(Erule)),
emit({indent(3),"'enc_",asn1ct_gen:list2name(Typename),
"_components'(Val",ObjFun,", [])"}),
emit({nl,"].",nl}),
@@ -453,6 +458,42 @@ gen_encode_sof(Erule,Typename,SeqOrSetOf,D) when is_record(D,type) ->
end,
gen_encode_sof_components(Erule,Typename,SeqOrSetOf,NewComponentType).
+
+%% Logic copied from asn1_per_bin_rt2ct:encode_constrained_number
+gen_encode_length({Lb,Ub},true) when Ub =< 65535, Lb >= 0 ->
+ Range = Ub - Lb + 1,
+ V2 = ["(length(Val) - ",Lb,")"],
+ Encode = if
+ Range == 1 ->
+ "[]";
+ Range == 2 ->
+ {"[",V2,"]"};
+ Range =< 4 ->
+ {"[10,2,",V2,"]"};
+ Range =< 8 ->
+ {"[10,3,",V2,"]"};
+ Range =< 16 ->
+ {"[10,4,",V2,"]"};
+ Range =< 32 ->
+ {"[10,5,",V2,"]"};
+ Range =< 64 ->
+ {"[10,6,",V2,"]"};
+ Range =< 128 ->
+ {"[10,7,",V2,"]"};
+ Range =< 255 ->
+ {"[10,8,",V2,"]"};
+ Range =< 256 ->
+ {"[20,1,",V2,"]"};
+ Range =< 65536 ->
+ {"[20,2,<<",V2,":16>>]"};
+ true ->
+ {"?RT_PER:encode_length(",{asis,{Lb,Ub}},",length(Val))"}
+ end,
+ emit({nl,Encode,",",nl});
+gen_encode_length(SizeConstraint,_) ->
+ emit({nl,indent(3),"?RT_PER:encode_length(",
+ {asis,SizeConstraint},",length(Val)),",nl}).
+
gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) ->
asn1ct_name:start(),
{_SeqOrSetOf,ComponentType} = D#type.def,
@@ -469,7 +510,8 @@ gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) ->
_ ->
""
end,
- emit({nl,"{Num,Bytes1} = ?RT_PER:decode_length(Bytes,",{asis,SizeConstraint},"),",nl}),
+ gen_decode_length(SizeConstraint,
+ is_optimized(Erules)),
emit({"'dec_",asn1ct_gen:list2name(Typename),
"_components'(Num, Bytes1, telltype",ObjFun,", []).",nl}),
NewComponentType =
@@ -480,6 +522,41 @@ gen_decode_sof(Erules,Typename,SeqOrSetOf,D) when is_record(D,type) ->
end,
gen_decode_sof_components(Erules,Typename,SeqOrSetOf,NewComponentType).
+%% Logic copied from asn1_per_bin_rt2ct:decode_constrained_number
+gen_decode_length({Lb,Ub},true) when Ub =< 65535, Lb >= 0 ->
+ Range = Ub - Lb + 1,
+ Call = if
+ Range == 1 ->
+ "{0,Bytes}";
+ Range == 2 ->
+ "?RT_PER:getbits(Bytes,1)";
+ Range =< 4 ->
+ "?RT_PER:getbits(Bytes,2)";
+ Range =< 8 ->
+ "?RT_PER:getbits(Bytes,3)";
+ Range =< 16 ->
+ "?RT_PER:getbits(Bytes,4)";
+ Range =< 32 ->
+ "?RT_PER:getbits(Bytes,5)";
+ Range =< 64 ->
+ "?RT_PER:getbits(Bytes,6)";
+ Range =< 128 ->
+ "?RT_PER:getbits(Bytes,7)";
+ Range =< 255 ->
+ "?RT_PER:getbits(Bytes,8)";
+ Range =< 256 ->
+ "?RT_PER:getoctets(Bytes,1)";
+ Range =< 65536 ->
+ "?RT_PER:getoctets(Bytes,2)";
+ true ->
+ ["exit({not_supported,{integer_range,",Range,"}}"]
+ end,
+ emit({nl,"{Val,Remain} = ",Call,",",nl}),
+ emit({nl,"{Num,Bytes1} = {Val+",Lb,",Remain},",nl});
+gen_decode_length(SizeConstraint,_) ->
+ emit({nl,"{Num,Bytes1} = ?RT_PER:decode_length(Bytes,",
+ {asis,SizeConstraint},"),",nl}).
+
gen_encode_sof_components(Erule,Typename,SeqOrSetOf,Cont) ->
{ObjFun,ObjFun_Var} =
case Cont#type.tablecinf of
@@ -636,6 +713,27 @@ gen_dec_extension_value(_) ->
emit({"{Ext,",{next,bytes},"} = ?RT_PER:getext(",{curr,bytes},")"}),
asn1ct_name:new(bytes).
+gen_fixoptionals([{Pos,Def}|R]) ->
+ asn1ct_name:new(fixopt),
+ emit({{curr,fixopt}," = case element(",{asis,Pos},",",{curr,val},") of",nl,
+ "asn1_DEFAULT -> 0;",nl,
+ {asis,Def}," -> 0;",nl,
+ "_ -> 1",nl,
+ "end,",nl}),
+ gen_fixoptionals(R);
+gen_fixoptionals([Pos|R]) ->
+ gen_fixoptionals([{Pos,asn1_NOVALUE}|R]);
+gen_fixoptionals([]) ->
+ ok.
+
+
+param_map(Fun, [H]) ->
+ [Fun(H)];
+param_map(Fun, [H|T]) ->
+ [Fun(H),","|param_map(Fun,T)].
+
+
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Produce a list with positions (in the Value record) where
%% there are optional components, start with 2 because first element
@@ -922,7 +1020,7 @@ gen_enc_line(Erule,TopType,Cname,Type,Element, _Pos,DynamicEnc,Ext) ->
end,
case Ext of
{ext,_Ep2,_} ->
- emit(["))"]);
+ emit("))");
_ -> true
end.
gen_dec_components_call(Erule,TopType,{Root1,ExtList,Root2},MaybeComma,DecInfObj,Ext,NumberOfOptionals) ->
diff --git a/lib/asn1/src/asn1ct_gen.erl b/lib/asn1/src/asn1ct_gen.erl
index e49829d82f..e18bc37058 100644
--- a/lib/asn1/src/asn1ct_gen.erl
+++ b/lib/asn1/src/asn1ct_gen.erl
@@ -47,6 +47,7 @@
un_hyphen_var/1]).
-export([gen_encode_constructed/4,
gen_decode_constructed/4]).
+-export([nif_parameter/0]).
%% pgen(Outfile, Erules, Module, TypeOrVal, Options)
%% Generate Erlang module (.erl) and (.hrl) file corresponding to an ASN.1 module
@@ -938,13 +939,13 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) ->
NoFinalPadding = lists:member(no_final_padding,get(encoding_options)),
Call = case Erules of
per -> "?RT_PER:complete(encode_disp(Type,Data))";
- per_bin -> "?RT_PER:complete(encode_disp(Type,Data))";
+ per_bin -> ["?RT_PER:complete(encode_disp(Type,Data))"];
ber -> "encode_disp(Type,Data)";
ber_bin -> "encode_disp(Type,Data)";
ber_bin_v2 -> "encode_disp(Type,Data)";
uper_bin when NoFinalPadding == true ->
"?RT_PER:complete_NFP(encode_disp(Type,Data))";
- uper_bin -> "?RT_PER:complete(encode_disp(Type,Data))"
+ uper_bin -> ["?RT_PER:complete(encode_disp(Type,Data))"]
end,
EncWrap = case Erules of
ber -> "wrap_encode(Bytes)";
@@ -974,7 +975,7 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) ->
% case Erules of
% ber_bin_v2 ->
% emit(["decode(Type,Data0) ->",nl]),
-% emit(["{Data,_RestBin} = ?RT_BER:decode(Data0",driver_parameter(),"),",nl]);
+% emit(["{Data,_RestBin} = ?RT_BER:decode(Data0",nif_parameter(),"),",nl]);
% _ ->
% emit(["decode(Type,Data) ->",nl])
% end,
@@ -991,10 +992,10 @@ pgen_dispatcher(Erules,_Module,{Types,_Values,_,_,_Objects,_ObjectSets}) ->
{ber_bin_v2,false} ->
io_lib:format("~s~s~s~n",
["element(1,?RT_BER:decode(Data",
- driver_parameter(),"))"]);
+ nif_parameter(),"))"]);
{ber_bin_v2,true} ->
emit(["{Data,Rest} = ?RT_BER:decode(Data0",
- driver_parameter(),"),",nl]),
+ nif_parameter(),"),",nl]),
"Data";
_ ->
"Data"
@@ -1130,13 +1131,8 @@ gen_decode_partial_incomplete(Erule) when Erule == ber;Erule==ber_bin;
"Data) of",nl]),
EmitCaseClauses(),
emit(["decode_part(Type,Data0) ->",nl]),
- Driver =
- case lists:member(driver,get(encoding_options)) of
- true ->
- ",driver";
- _ -> ""
- end,
- emit([" case catch decode_inc_disp(Type,element(1,?RT_BER:decode(Data0",Driver,"))) of",nl]),
+ emit([" case catch decode_inc_disp(Type,element(1,"
+ "?RT_BER:decode(Data0",nif_parameter(),"))) of",nl]),
% " {Data,_RestBin} = ?RT_BER:decode(Data0),",nl,
% " case catch decode_inc_disp(Type,Data) of",nl]),
EmitCaseClauses();
@@ -1179,12 +1175,12 @@ gen_partial_inc_dispatcher([],_) ->
emit(["decode_partial_inc_disp(Type,_Data) ->",nl,
" exit({error,{asn1,{undefined_type,Type}}}).",nl]).
-driver_parameter() ->
+nif_parameter() ->
Options = get(encoding_options),
- case lists:member(driver,Options) of
- true ->
- ",driver";
- _ -> ""
+ case {lists:member(driver,Options),lists:member(nif,Options)} of
+ {true,_} -> ",nif";
+ {_,true} -> ",nif";
+ _ -> ""
end.
gen_wrapper() ->
diff --git a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl
index 9ec458e351..e8a4ad0cf1 100644
--- a/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl
+++ b/lib/asn1/src/asn1ct_gen_ber_bin_v2.erl
@@ -416,7 +416,7 @@ gen_decode_selected(Erules,Type,FuncName) ->
end,
emit([" case ?RT_BER:decode_selective(",{asis,Pattern},",Bin) of",nl,
" {ok,Bin2} when is_binary(Bin2) ->",nl,
- " {Tlv,_} = ?RT_BER:decode(Bin2),",nl]),
+ " {Tlv,_} = ?RT_BER:decode(Bin2",asn1ct_gen:nif_parameter(),"),",nl]),
emit("{ok,"),
gen_decode_selected_type(Erules,Type),
emit(["};",nl," Err -> exit({error,{selctive_decode,Err}})",nl,
@@ -708,7 +708,7 @@ gen_dec_prim(Erules,Att,BytesVar,DoTag,TagIn,Form,OptOrMand) ->
'ASN1_OPEN_TYPE' ->
emit(["?RT_BER:decode_open_type_as_binary(",
BytesVar,","]),
- add_func({decode_open_type_as_binary,2});
+ add_func({decode_open_type_as_binary,3});
#'ObjectClassFieldType'{} ->
case asn1ct_gen:get_inner(Att#type.def) of
{fixedtypevaluefield,_,InnerType} ->
@@ -716,7 +716,7 @@ gen_dec_prim(Erules,Att,BytesVar,DoTag,TagIn,Form,OptOrMand) ->
'ASN1_OPEN_TYPE' ->
emit(["?RT_BER:decode_open_type_as_binary(",
BytesVar,","]),
- add_func({decode_open_type_as_binary,2});
+ add_func({decode_open_type_as_binary,3});
Other ->
exit({'can not decode' ,Other})
end;
@@ -1064,7 +1064,7 @@ emit_tlv_format_function() ->
end.
emit_tlv_format_function1() ->
emit(["tlv_format(Bytes) when is_binary(Bytes) ->",nl,
- " {Tlv,_}=?RT_BER:decode(Bytes),",nl,
+ " {Tlv,_}=?RT_BER:decode(Bytes",asn1ct_gen:nif_parameter(),"),",nl,
" Tlv;",nl,
"tlv_format(Bytes) ->",nl,
" Bytes.",nl]).
diff --git a/lib/asn1/src/asn1ct_gen_per.erl b/lib/asn1/src/asn1ct_gen_per.erl
index 8313cf1b60..b90a0adf81 100644
--- a/lib/asn1/src/asn1ct_gen_per.erl
+++ b/lib/asn1/src/asn1ct_gen_per.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -238,7 +238,8 @@ gen_encode_prim(Erules,D,DoTag,Value) when is_record(D,type) ->
"?RT_PER:complete(enc_~s(~s))",[Tname,Value]);
[#type{def=#'Externaltypereference'{type=Tname}}] ->
io_lib:format(
- "?RT_PER:complete(enc_~s(~s))",[Tname,Value]);
+ "?RT_PER:complete(enc_~s(~s))",
+ [Tname,Value]);
_ -> Value
end,
emit(["?RT_PER:encode_open_type(", {asis,Constraint}, ",",
diff --git a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl
index 4f4fcfafc3..1a0a0e211d 100644
--- a/lib/asn1/src/asn1ct_gen_per_rt2ct.erl
+++ b/lib/asn1/src/asn1ct_gen_per_rt2ct.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2002-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -230,7 +230,8 @@ gen_encode_prim(Erules,D,DoTag,Value) when is_record(D,type) ->
"?RT_PER:complete(enc_~s(~s))",[Tname,Value]);
[#type{def=#'Externaltypereference'{type=Tname}}] ->
io_lib:format(
- "?RT_PER:complete(enc_~s(~s))",[Tname,Value]);
+ "?RT_PER:complete(enc_~s(~s))",
+ [Tname,Value]);
_ -> Value
end,
emit(["?RT_PER:encode_open_type(", {asis,Constraint}, ",",
diff --git a/lib/asn1/src/asn1rt.erl b/lib/asn1/src/asn1rt.erl
index 9ef68efab5..e9d3ea9a72 100644
--- a/lib/asn1/src/asn1rt.erl
+++ b/lib/asn1/src/asn1rt.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -27,6 +27,8 @@
-export([utf8_binary_to_list/1,utf8_list_to_binary/1]).
+-deprecated([load_driver/0,unload_driver/0]).
+
encode(Module,{Type,Term}) ->
encode(Module,Type,Term).
@@ -46,38 +48,12 @@ decode(Module,Type,Bytes) ->
Result
end.
-%% asn1-1.6.8.1
-%% load_driver() ->
-%% asn1rt_driver_handler:load_driver(),
-%% receive
-%% driver_ready ->
-%% ok;
-%% Err={error,_Reason} ->
-%% Err;
-%% Error ->
-%% {error,Error}
-%% end.
-
-%% asn1-1.6.9
- load_driver() ->
- case catch asn1rt_driver_handler:load_driver() of
- ok ->
- ok;
- {error,{already_started,asn1}} ->
- ok;
- Err ->
- {error,Err}
- end.
-
+%% Remove in R16A
+load_driver() ->
+ ok.
unload_driver() ->
- case catch asn1rt_driver_handler:unload_driver() of
- ok ->
- ok;
- Error ->
- {error,Error}
- end.
-
+ ok.
info(Module) ->
case catch apply(Module,info,[]) of
diff --git a/lib/asn1/src/asn1rt_ber_bin_v2.erl b/lib/asn1/src/asn1rt_ber_bin_v2.erl
index a3bb570282..2df289e2b8 100644
--- a/lib/asn1/src/asn1rt_ber_bin_v2.erl
+++ b/lib/asn1/src/asn1rt_ber_bin_v2.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -49,7 +49,8 @@
decode_tag_and_length/1]).
-export([encode_open_type/1,encode_open_type/2,
- decode_open_type/2,decode_open_type_as_binary/2]).
+ decode_open_type/2,decode_open_type/3,
+ decode_open_type_as_binary/2]).
-export([decode_primitive_incomplete/2,decode_selective/2]).
@@ -158,64 +159,47 @@ encode_tlv_list([],Acc) ->
Bin=list_to_binary(lists:reverse(Acc)),
{Bin,size(Bin)}.
-%% asn1-1.6.8.1
-%% decode(B,driver) ->
-%% case catch port_control(asn1_driver_port,2,B) of
-%% Bin when is_binary(Bin) ->
-%% binary_to_term(Bin);
-%% List when is_list(List) -> handle_error(List,B);
-%% {'EXIT',{badarg,Reason}} ->
-%% asn1rt_driver_handler:load_driver(),
-%% receive
-%% driver_ready ->
-%% case catch port_control(asn1_driver_port,2,B) of
-%% Bin2 when is_binary(Bin2) -> binary_to_term(Bin2);
-%% List when is_list(List) -> handle_error(List,B);
-%% Error -> exit(Error)
-%% end;
-%% {error,Error} -> % error when loading driver
-%% %% the driver could not be loaded
-%% exit(Error);
-%% Error={port_error,Reason} ->
-%% exit(Error)
-%% end;
-%% {'EXIT',Reason} ->
-%% exit(Reason)
-%% end.
-
-%% asn1-1.6.9
-decode(B,driver) ->
- case catch control(?TLV_DECODE,B) of
- Bin when is_binary(Bin) ->
- binary_to_term(Bin);
- List when is_list(List) -> handle_error(List,B);
- {'EXIT',{badarg,_Reason}} ->
- case asn1rt:load_driver() of
- ok ->
- case control(?TLV_DECODE,B) of
- Bin when is_binary(Bin) -> binary_to_term(Bin);
- List when is_list(List) -> handle_error(List,B)
- end;
- Err ->
- Err
- end
- end.
+decode(B) ->
+ decode(B, erlang).
+%% asn1-1.7
+decode(B, nif) ->
+ case application:get_env(asn1, nif_loadable) of
+ {ok, true} ->
+ case asn1rt_nif:decode_ber_tlv(B) of
+ {error, Reason} -> handle_error(Reason, B);
+ Else -> Else
+ end;
+ {ok, false} ->
+ decode(B);
+ undefined ->
+ case catch code:load_file(asn1rt_nif) of
+ {module, asn1rt_nif} ->
+ application:set_env(asn1, nif_loadable, true);
+ _Else ->
+ application:set_env(asn1, nif_loadable, false)
+ end,
+ decode(B, nif)
+ end;
+decode(B,erlang) when is_binary(B) ->
+ decode_primitive(B);
+decode(Tlv,erlang) ->
+ {Tlv,<<>>}.
handle_error([],_)->
exit({error,{asn1,{"memory allocation problem"}}});
-handle_error([$1|_],L) -> % error in driver
+handle_error({$1,_},L) -> % error in nif
exit({error,{asn1,L}});
-handle_error([$2|T],L) -> % error in driver due to wrong tag
+handle_error({$2,T},L) -> % error in nif due to wrong tag
exit({error,{asn1,{"bad tag after byte:",error_pos(T),L}}});
-handle_error([$3|T],L) -> % error in driver due to length error
+handle_error({$3,T},L) -> % error in driver due to length error
exit({error,{asn1,{"bad length field after byte:",
error_pos(T),L}}});
-handle_error([$4|T],L) -> % error in driver due to indefinite length error
+handle_error({$4,T},L) -> % error in driver due to indefinite length error
exit({error,{asn1,
{"indefinite length without end bytes after byte:",
error_pos(T),L}}});
-handle_error([$5|T],L) -> % error in driver due to indefinite length error
+handle_error({$5,T},L) -> % error in driver due to indefinite length error
exit({error,{asn1,{"bad encoded value after byte:",
error_pos(T),L}}});
handle_error(ErrL,L) ->
@@ -228,16 +212,6 @@ error_pos([B])->
error_pos([B|Bs]) ->
BS = 8 * length(Bs),
B bsl BS + error_pos(Bs).
-%% asn1-1.6.9
-control(Cmd, Data) ->
- Port = asn1rt_driver_handler:client_port(),
- erlang:port_control(Port, Cmd, Data).
-
-decode(Bin) when is_binary(Bin) ->
- decode_primitive(Bin);
-decode(Tlv) -> % assume it is a tlv
- {Tlv,<<>>}.
-
decode_primitive(Bin) ->
{Form,TagNo,V,Rest} = decode_tag_and_length(Bin),
@@ -796,9 +770,11 @@ encode_open_type(Val,Tag) ->
%% Value = binary with decoded data (which must be decoded again as some type)
%%
decode_open_type(Tlv, TagIn) ->
+ decode_open_type(Tlv, TagIn, erlang).
+decode_open_type(Tlv, TagIn, Method) ->
case match_tags(Tlv,TagIn) of
Bin when is_binary(Bin) ->
- {InnerTlv,_} = decode(Bin),
+ {InnerTlv,_} = decode(Bin,Method),
InnerTlv;
TlvBytes -> TlvBytes
end.
@@ -1577,14 +1553,12 @@ e_object_identifier(V) when is_tuple(V) ->
e_object_identifier([E1, E2 | Tail]) ->
Head = 40*E1 + E2, % wow!
{H,Lh} = mk_object_val(Head),
- {R,Lr} = enc_obj_id_tail(Tail, [], 0),
+ {R,Lr} = lists:mapfoldl(fun enc_obj_id_tail/2,0,Tail),
{[H|R], Lh+Lr}.
-enc_obj_id_tail([], Ack, Len) ->
- {lists:reverse(Ack), Len};
-enc_obj_id_tail([H|T], Ack, Len) ->
+enc_obj_id_tail(H, Len) ->
{B, L} = mk_object_val(H),
- enc_obj_id_tail(T, [B|Ack], Len+L).
+ {B,Len+L}.
%%%%%%%%%%%
diff --git a/lib/asn1/src/asn1rt_driver_handler.erl b/lib/asn1/src/asn1rt_driver_handler.erl
deleted file mode 100644
index 146d0043f9..0000000000
--- a/lib/asn1/src/asn1rt_driver_handler.erl
+++ /dev/null
@@ -1,144 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
-%%
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%%
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%%
-%% %CopyrightEnd%
-%%
-%%
-
--module(asn1rt_driver_handler).
-
--include("asn1_records.hrl").
-
--export([load_driver/0,unload_driver/0,client_port/0]).
-
-%% Internal exports
--export([init/2]).
-
-%% Macros
--define(port_names,
- { asn1_drv01, asn1_drv02, asn1_drv03, asn1_drv04,
- asn1_drv05, asn1_drv06, asn1_drv07, asn1_drv08,
- asn1_drv09, asn1_drv10, asn1_drv11, asn1_drv12,
- asn1_drv13, asn1_drv14, asn1_drv15, asn1_drv16 }).
-
-%%% --------------------------------------------------------
-%%% Interface Functions.
-%%% --------------------------------------------------------
-load_driver() ->
- load_driver(noreason).
-
-load_driver(Reason) ->
- Ref = make_ref(),
- case whereis(asn1_driver_owner) of % to prevent unnecessary spawn
- Pid when is_pid(Pid) ->
- asn1_driver_owner ! {self(),Ref,are_you_ready},
- receive
- {Ref,driver_ready} ->
- ok
- after 10000 ->
- {error,{timeout,waiting_for_drivers}}
- end;
- _ ->
- {_,Mref} = spawn_monitor(asn1rt_driver_handler, init, [self(),Ref]),
- receive
- {'DOWN', Mref, _, _, NewReason} ->
- case NewReason of
- Reason -> {error,Reason};
- _ -> load_driver(NewReason)
- end;
- {Ref,driver_ready} ->
- erlang:demonitor(Mref),
- ok;
- {Ref,Error = {error,_Reason}} ->
- erlang:demonitor(Mref),
- Error
- after 10000 -> %% 10 seconds
- {error,{timeout,waiting_for_drivers}}
- end
- end.
-
-init(FromPid,FromRef) ->
- case catch register(asn1_driver_owner,self()) of
- true -> true;
- _Other -> exit(normal)
- end,
- Dir = filename:join([code:priv_dir(asn1),"lib"]),
- case catch erl_ddll:load_driver(Dir,asn1_erl_drv) of
- ok ->
- Result = open_named_ports(),
- catch (FromPid ! {FromRef,Result}),
- loop(Result);
- {error,Err} -> % if erl_ddll:load_driver fails
- ForErr = erl_ddll:format_error(Err),
- OSDir = filename:join(Dir,erlang:system_info(system_architecture)),
- case catch erl_ddll:load_driver(OSDir,asn1_erl_drv) of
- ok ->
- Result = open_named_ports(),
- catch (FromPid ! {FromRef,Result}),
- loop(Result);
- {error,Err2} ->
-% catch (FromPid ! {FromRef,Error})
- ForErr2 = erl_ddll:format_error(Err2),
- catch (FromPid ! {FromRef,{error,{{Dir,ForErr},{OSDir,ForErr2}}}})
- end
- end.
-
-
-open_named_ports() ->
- open_named_ports(size(?port_names)).
-
-open_named_ports(0) ->
- driver_ready;
-open_named_ports(N) ->
- case catch open_port({spawn,"asn1_erl_drv"},[]) of
- {'EXIT',Reason} ->
- {error,{port_error,Reason}};
- Port ->
- register(element(N,?port_names),Port),
- open_named_ports(N-1)
- end.
-
-loop(Result) ->
- receive
- {_FromPid,_FromRef,unload} ->
- close_ports(size(?port_names)),
- erl_ddll:unload_driver(asn1_erl_drv),
- ok;
- {FromPid,FromRef,are_you_ready} ->
- catch (FromPid ! {FromRef,driver_ready}),
- loop(Result);
- _ ->
- loop(Result)
- end.
-
-unload_driver() ->
- case whereis(asn1_driver_owner) of
- Pid when is_pid(Pid) ->
- Pid ! {self(),make_ref(),unload},
- ok;
- _ ->
- ok
- end.
-
-close_ports(0) ->
- ok;
-close_ports(N) ->
- element(N,?port_names) ! {self(), close}, %% almost same as port_close(Name)
- close_ports(N-1).
-
-client_port() ->
- element(erlang:system_info(scheduler_id) rem size(?port_names) + 1,
- ?port_names).
diff --git a/lib/asn1/src/asn1rt_nif.erl b/lib/asn1/src/asn1rt_nif.erl
new file mode 100644
index 0000000000..8580c70e6b
--- /dev/null
+++ b/lib/asn1/src/asn1rt_nif.erl
@@ -0,0 +1,83 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%
+-module(asn1rt_nif).
+
+%% Nif interface for asn1
+
+-export([encode_per_complete/1,
+ decode_ber_tlv/1]).
+
+-on_load(load_nif/0).
+
+-define(ASN1_NIF_VSN,1).
+
+load_nif() ->
+ LibBaseName = "asn1_erl_nif",
+ PrivDir = code:priv_dir(asn1),
+ LibName = case erlang:system_info(build_type) of
+ opt ->
+ LibBaseName;
+ Type ->
+ LibTypeName = LibBaseName ++ "." ++ atom_to_list(Type),
+ case (filelib:wildcard(
+ filename:join(
+ [PrivDir,
+ "lib",
+ LibTypeName ++ "*"])) /= []) orelse
+ (filelib:wildcard(
+ filename:join(
+ [PrivDir,
+ "lib",
+ erlang:system_info(system_architecture),
+ LibTypeName ++ "*"])) /= []) of
+ true -> LibTypeName;
+ false -> LibBaseName
+ end
+ end,
+ Lib = filename:join([PrivDir, "lib", LibName]),
+ Status = case erlang:load_nif(Lib, ?ASN1_NIF_VSN) of
+ ok -> ok;
+ {error, {load_failed, _}}=Error1 ->
+ ArchLibDir =
+ filename:join([PrivDir, "lib",
+ erlang:system_info(system_architecture)]),
+ Candidate =
+ filelib:wildcard(filename:join([ArchLibDir,LibName ++ "*" ])),
+ case Candidate of
+ [] -> Error1;
+ _ ->
+ ArchLib = filename:join([ArchLibDir, LibName]),
+ erlang:load_nif(ArchLib, ?ASN1_NIF_VSN)
+ end;
+ Error1 -> Error1
+ end,
+ case Status of
+ ok -> ok;
+ {error, {E, Str}} ->
+ error_logger:error_msg("Unable to load asn1 nif library. "
+ "Failed with error:~n\"~p, ~s\"~n",[E,Str]),
+ Status
+ end.
+
+encode_per_complete(_Binary) ->
+ erlang:nif_error({nif_not_loaded,module,?MODULE,line,?LINE}).
+
+decode_ber_tlv(_Binary) ->
+ erlang:nif_error({nif_not_loaded,module,?MODULE,line,?LINE}).
diff --git a/lib/asn1/src/asn1rt_per_bin.erl b/lib/asn1/src/asn1rt_per_bin.erl
index 6bbca26209..a124c7553d 100644
--- a/lib/asn1/src/asn1rt_per_bin.erl
+++ b/lib/asn1/src/asn1rt_per_bin.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -57,7 +57,7 @@
encode_NumericString/2, decode_NumericString/2,
encode_ObjectDescriptor/2, decode_ObjectDescriptor/1
]).
--export([complete_bytes/1]).
+-export([complete_bytes/1, getbits/2, getoctets/2]).
-define('16K',16384).
-define('32K',32768).
diff --git a/lib/asn1/src/asn1rt_per_bin_rt2ct.erl b/lib/asn1/src/asn1rt_per_bin_rt2ct.erl
index f4aecf9322..c7ead680ce 100644
--- a/lib/asn1/src/asn1rt_per_bin_rt2ct.erl
+++ b/lib/asn1/src/asn1rt_per_bin_rt2ct.erl
@@ -1734,143 +1734,24 @@ get_constraint(C,Key) ->
-ifdef(nodriver).
complete(L) ->
- case complete1(L) of
- {[],[]} ->
- <<0>>;
- {Acc,[]} ->
- Acc;
- {Acc,Bacc} ->
- [Acc|complete_bytes(Bacc)]
- end.
-
-
-% this function builds the ugly form of lists [E1|E2] to avoid having to reverse it at the end.
-% this is done because it is efficient and that the result always will be sent on a port or
-% converted by means of list_to_binary/1
- complete1(InList) when is_list(InList) ->
- complete1(InList,[],[]);
- complete1(InList) ->
- complete1([InList],[],[]).
-
- complete1([],Acc,Bacc) ->
- {Acc,Bacc};
- complete1([H|T],Acc,Bacc) when is_list(H) ->
- {NewH,NewBacc} = complete1(H,Acc,Bacc),
- complete1(T,NewH,NewBacc);
-
- complete1([{octets,Bin}|T],Acc,[]) ->
- complete1(T,[Acc|Bin],[]);
-
- complete1([{octets,Bin}|T],Acc,Bacc) ->
- complete1(T,[Acc|[complete_bytes(Bacc),Bin]],[]);
-
- complete1([{debug,_}|T], Acc,Bacc) ->
- complete1(T,Acc,Bacc);
-
- complete1([{bits,N,Val}|T],Acc,Bacc) ->
- complete1(T,Acc,complete_update_byte(Bacc,Val,N));
-
- complete1([{bit,Val}|T],Acc,Bacc) ->
- complete1(T,Acc,complete_update_byte(Bacc,Val,1));
-
- complete1([align|T],Acc,[]) ->
- complete1(T,Acc,[]);
- complete1([align|T],Acc,Bacc) ->
- complete1(T,[Acc|complete_bytes(Bacc)],[]);
- complete1([{0,Bin}|T],Acc,[]) when is_binary(Bin) ->
- complete1(T,[Acc|Bin],[]);
- complete1([{Unused,Bin}|T],Acc,[]) when is_integer(Unused),is_binary(Bin) ->
- Size = size(Bin)-1,
- <<Bs:Size/binary,B>> = Bin,
- NumBits = 8-Unused,
- complete1(T,[Acc|Bs],[[B bsr Unused]|NumBits]);
- complete1([{Unused,Bin}|T],Acc,Bacc) when is_integer(Unused),is_binary(Bin) ->
- Size = size(Bin)-1,
- <<Bs:Size/binary,B>> = Bin,
- NumBits = 8 - Unused,
- Bf = complete_bytes(Bacc),
- complete1(T,[Acc|[Bf,Bs]],[[B bsr Unused]|NumBits]).
-
-
- complete_update_byte([],Val,Len) ->
- complete_update_byte([[0]|0],Val,Len);
- complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) when NumBits + Len == 8 ->
- [[0,((Byte bsl Len) + Val) band 255|Bacc]|0];
- complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) when NumBits + Len > 8 ->
- Rem = 8 - NumBits,
- Rest = Len - Rem,
- complete_update_byte([[0,((Byte bsl Rem) + (Val bsr Rest)) band 255 |Bacc]|0],Val,Rest);
- complete_update_byte([[Byte|Bacc]|NumBits],Val,Len) ->
- [[((Byte bsl Len) + Val) band 255|Bacc]|NumBits+Len].
-
-
- complete_bytes([[Byte|Bacc]|0]) ->
- lists:reverse(Bacc);
- complete_bytes([[Byte|Bacc]|NumBytes]) ->
- lists:reverse([(Byte bsl (8-NumBytes)) band 255|Bacc]);
- complete_bytes([]) ->
- [].
+ erlang_complete(L).
-else.
-%% asn1-1.6.8.1_dev
-%% complete(L) ->
-%% case catch port_control(asn1_driver_port,1,L) of
-%% Bin when is_binary(Bin) ->
-%% Bin;
-%% List when is_list(List) -> handle_error(List,L);
-%% {'EXIT',{badarg,Reason}} ->
-%% asn1rt_driver_handler:load_driver(),
-%% receive
-%% driver_ready ->
-%% case catch port_control(asn1_driver_port,1,L) of
-%% Bin2 when is_binary(Bin2) -> Bin2;
-%% List when is_list(List) -> handle_error(List,L);
-%% {'EXIT',Reason2={badarg,_R}} ->
-%% exit({"failed to call driver probably due to bad asn1 value",Reason2});
-%% Reason2 -> exit(Reason2)
-%% end;
-%% {error,Error} -> % error when loading driver
-%% %% the driver could not be loaded
-%% exit(Error);
-%% Error={port_error,Reason} ->
-%% exit(Error)
-%% end;
-%% {'EXIT',Reason} ->
-%% exit(Reason)
-%% end.
-
-%% asn1-1.6.9
+%% asn1-1.7
complete(L) ->
- case catch control(?COMPLETE_ENCODE,L) of
- Bin when is_binary(Bin) ->
- Bin;
- List when is_list(List) -> handle_error(List,L);
- {'EXIT',{badarg,_Reason}} ->
- case asn1rt:load_driver() of
- ok ->
- case control(?COMPLETE_ENCODE,L) of
- Bin when is_binary(Bin) ->Bin;
- List when is_list(List) -> handle_error(List,L)
- end;
- Err ->
- Err
- end
+ case asn1rt_nif:encode_per_complete(L) of
+ {error, Reason} -> handle_error(Reason, L);
+ Else when is_binary(Else) -> Else
end.
-
handle_error([],_)->
exit({error,{asn1,{"memory allocation problem in driver"}}});
-handle_error("1",L) -> % error in complete in driver
+handle_error($1,L) -> % error in complete in driver
exit({error,{asn1,L}});
handle_error(ErrL,L) ->
exit({error,{asn1,ErrL,L}}).
-%% asn1-1.6.9
-control(Cmd, Data) ->
- Port = asn1rt_driver_handler:client_port(),
- erlang:port_control(Port, Cmd, Data).
-
-endif.
diff --git a/lib/asn1/test/asn1.cover b/lib/asn1/test/asn1.cover
index 589a8b7e3d..ad3a0f3db9 100644
--- a/lib/asn1/test/asn1.cover
+++ b/lib/asn1/test/asn1.cover
@@ -1,2 +1,3 @@
{incl_app,asn1,details}.
+{excl_mods, asn1, [asn1rt_nif]}. \ No newline at end of file
diff --git a/lib/asn1/test/asn1_SUITE.erl.src b/lib/asn1/test/asn1_SUITE.erl.src
index 582ccd877c..5339ad78c7 100644
--- a/lib/asn1/test/asn1_SUITE.erl.src
+++ b/lib/asn1/test/asn1_SUITE.erl.src
@@ -2036,11 +2036,7 @@ rtUI(Config) ->
?line {ok,_} = asn1rt:info('Prim'),
?line ok = asn1ct:compile(filename:join(DataDir,"Prim"),[?PER]),
- ?line {ok,_} = asn1rt:info('Prim'),
-
- ?line ok = asn1rt:load_driver(),
- ?line ok = asn1rt:load_driver(),
- ?line ok = asn1rt:unload_driver().
+ ?line {ok,_} = asn1rt:info('Prim').
testROSE(suite) -> [];
testROSE(Config) ->
diff --git a/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src b/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src
index abd21b0d78..6e15aa9fdc 100644
--- a/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src
+++ b/lib/asn1/test/asn1_bin_v2_particular_SUITE.erl.src
@@ -1,5 +1,5 @@
-particular() -> [smp, ticket7904].
+particular() -> [smp, per_performance, ber_performance, ticket7904].
smp(suite) -> [];
@@ -11,37 +11,137 @@ smp(Config) ->
?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()},
?line ok = testNBAPsystem:compile(Config,per_bin,[optimize]),
-
- Parent = self(),
- ?line ok = asn1rt:load_driver(),
-
- smp2(Parent,NumOfProcs,Msg,2),
+ enc_dec(NumOfProcs,Msg,2),
N = 10000,
- ?line {Time1,ok} = timer:tc(?MODULE,smp2,[Parent,NumOfProcs,Msg, N]),
- ?line {Time1S,ok} = timer:tc(?MODULE,sequential,[NumOfProcs * N,Msg]),
+ ?line {Time1,ok} = timer:tc(?MODULE,enc_dec,[NumOfProcs,Msg, N]),
+ ?line {Time1S,ok} = timer:tc(?MODULE,enc_dec,[1, Msg, NumOfProcs * N]),
- ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,driver]),
- ?line {Time2,ok} = timer:tc(?MODULE,smp2,[Parent,NumOfProcs,Msg, N]),
+ ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,nif]),
+ ?line {Time3,ok} = timer:tc(?MODULE,enc_dec,[NumOfProcs,Msg, N]),
- ?line {Time2S,ok} = timer:tc(?MODULE,sequential,[NumOfProcs * N,Msg]),
+ ?line {Time3S,ok} = timer:tc(?MODULE,enc_dec,[1, Msg, NumOfProcs * N]),
- {comment,lists:flatten(io_lib:format("Encode/decode time parallell with ~p cores: ~p [microsecs]~nEncode/decode time sequential: ~p [microsecs]",[NumOfProcs,Time1+Time2,Time1S+Time2S]))};
+ {comment,lists:flatten(
+ io_lib:format(
+ "Encode/decode time parallell with ~p cores: ~p [microsecs]~n"
+ "Encode/decode time sequential: ~p [microsecs]",
+ [NumOfProcs,Time1+Time3,Time1S+Time3S]))};
false ->
{skipped,"No smp support"}
end.
-smp2(Parent,NumOfProcs,Msg, N) ->
- Pids = [spawn_link(fun() -> worker(Msg,Parent, N) end)
- || _ <- lists:seq(1,NumOfProcs)],
- ?line ok = wait_pids(Pids).
+per_performance(Config) ->
+
+ ?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()},
+ ?line ok = testNBAPsystem:compile(Config,per_bin,[optimize]),
+ NumOfProcs = erlang:system_info(schedulers)*10,
+ N = 10000,
+ SmpN = lists:seq(1,round(N/NumOfProcs)),
+
+ asn1_wrapper:encode('NBAP-PDU-Discriptions','NBAP-PDU', Msg),
+
+ PerFun = fun() ->
+ [asn1_wrapper:encode('NBAP-PDU-Discriptions',
+ 'NBAP-PDU',
+ Msg) || _I <- lists:seq(1,N)],
+ ok
+ end,
+
+ PerSMPFun =
+ fun() ->
+ pforeach(fun(_) ->
+ [asn1_wrapper:encode('NBAP-PDU-Discriptions',
+ 'NBAP-PDU',
+ Msg) || _I <- SmpN]
+ end,lists:seq(1,NumOfProcs))
+ end,
+
+ ?line {TimeN,ok} = timer:tc(PerFun),
+ ?line {TimeNS,ok} = timer:tc(PerSMPFun),
+
+ ?line ok = testNBAPsystem:compile(Config,per_bin,[]),
+
+ ?line {TimeE,ok} = timer:tc(PerFun),
+ ?line {TimeES,ok} = timer:tc(PerSMPFun),
+
+ ct:log("Seq:<br/>"
+ "Nif : ~p (~.2f%)<br/>"
+ "Erlang: ~p (~.2f%)<br/>"
+ "Parallel:<br/>"
+ "Nif : ~p (~.2f%)<br/>"
+ "Erlang: ~p (~.2f%)<br/>",
+ [TimeN,TimeN/TimeN*100,
+ TimeE,TimeE/TimeN*100,
+ TimeNS,TimeNS/TimeNS*100,
+ TimeES,TimeES/TimeNS*100]),
+
+ {comment, lists:flatten(io_lib:format("Nifs are ~.2f% faster than erlang!",
+ [faster(TimeN+TimeNS,
+ TimeE+TimeES)]))}.
+
+ber_performance(Config) ->
+
+ ?line Msg = {initiatingMessage, testNBAPsystem:cell_setup_req_msg()},
+ ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize,nif]),
+ NumOfProcs = erlang:system_info(schedulers)*10,
+ N = 10000,
+ SmpN = lists:seq(1,round(N/NumOfProcs)),
+
+ {ok,B} = asn1_wrapper:encode('NBAP-PDU-Discriptions','NBAP-PDU', Msg),
+
+ BerFun = fun() ->
+ [asn1_wrapper:decode(
+ 'NBAP-PDU-Discriptions',
+ 'NBAP-PDU',
+ B) || _I <- lists:seq(1,N)],
+ ok
+ end,
+ BerSMPFun =
+ fun() ->
+ pforeach(fun(_) ->
+ [asn1_wrapper:decode(
+ 'NBAP-PDU-Discriptions',
+ 'NBAP-PDU',
+ B) || _I <- SmpN]
+ end,lists:seq(1,NumOfProcs))
+ end,
+
+ ?line {TimeN,ok} = timer:tc(BerFun),
+ ?line {TimeNS,ok} = timer:tc(BerSMPFun),
+
+ ?line ok = testNBAPsystem:compile(Config,ber_bin,[optimize]),
+
+ ?line {TimeE,ok} = timer:tc(BerFun),
+ ?line {TimeES,ok} = timer:tc(BerSMPFun),
+
+ ct:log("Seq:<br/>"
+ "Nif : ~p (~.2f%)<br/>"
+ "Erlang: ~p (~.2f%)<br/>"
+ "Parallel:<br/>"
+ "Nif : ~p (~.2f%)<br/>"
+ "Erlang: ~p (~.2f%)<br/>",
+ [TimeN,TimeN/TimeN*100,
+ TimeE,TimeE/TimeN*100,
+ TimeNS,TimeNS/TimeNS*100,
+ TimeES,TimeES/TimeNS*100]),
+
+ {comment, lists:flatten(io_lib:format("Nifs are ~.2f% faster than erlang!",
+ [faster(TimeN+TimeNS,
+ TimeE+TimeES)]))}.
-worker(Msg, Parent, N) ->
- %% io:format("smp worker ~p with ~p worker loops.~n",[self(), N]),
- worker_loop(N, Msg),
- Parent ! self().
+
+faster(A,B) ->
+ (B - A)/B * 100.
+
+enc_dec(1, Msg, N) ->
+ worker_loop(N, Msg);
+enc_dec(NumOfProcs,Msg, N) ->
+ pforeach(fun(_) ->
+ worker_loop(N, Msg)
+ end, [I || I <- lists:seq(1,NumOfProcs)]).
worker_loop(0, _Msg) ->
ok;
@@ -50,28 +150,24 @@ worker_loop(N, Msg) ->
'NBAP-PDU',
Msg),
?line {ok,_Msg}=asn1_wrapper:decode('NBAP-PDU-Discriptions',
- 'NBAP-PDU',
- B),
+ 'NBAP-PDU',
+ B),
worker_loop(N - 1, Msg).
-wait_pids([]) ->
- ok;
-wait_pids(Pids) ->
+pforeach(Fun, List) ->
+ pforeach(Fun, List, []).
+pforeach(Fun, [], [{Pid,Ref}|Pids]) ->
receive
- Pid when is_pid(Pid) ->
- ?line true = lists:member(Pid,Pids),
- Others = lists:delete(Pid,Pids),
- io:format("wait_pid got ~p, still waiting for ~p\n",[Pid,Others]),
- wait_pids(Others);
- Err ->
- io:format("Err: ~p~n",[Err]),
- ?line exit(Err)
- end.
-
-sequential(N,Msg) ->
- %%io:format("sequential encode/decode with N = ~p~n",[N]),
- worker_loop(N,Msg).
+ {'DOWN', Ref, process, Pid, normal} ->
+ pforeach(Fun, [], Pids)
+ end;
+pforeach(Fun, [H|T], Pids) ->
+ Pid = spawn(fun() -> Fun(H) end),
+ Ref = erlang:monitor(process, Pid),
+ pforeach(Fun, T, [{Pid, Ref}|Pids]);
+pforeach(_Fun,[],[]) ->
+ ok.
-record('InitiatingMessage',{procedureCode,criticality,value}).
-record('Iu-ReleaseCommand',{first,second}).
diff --git a/lib/asn1/test/ber_decode_error.erl b/lib/asn1/test/ber_decode_error.erl
index 96d6545636..a566e0b07f 100644
--- a/lib/asn1/test/ber_decode_error.erl
+++ b/lib/asn1/test/ber_decode_error.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1999-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -45,6 +45,10 @@ run([]) ->
run([driver]) ->
%% test of OTP-4797, bad indata to driver does not cause an EXIT
?line {error,_Reason} = asn1rt:decode('Constructed','S3',[3,5]),
+ ok;
+run([nif]) ->
+ %% test of OTP-4797, bad indata to driver does not cause an EXIT
+ ?line {error,_Reason} = asn1rt:decode('Constructed','S3',[3,5]),
ok.
diff --git a/lib/asn1/test/testPrim.erl b/lib/asn1/test/testPrim.erl
index 97f99e7b1c..39c1e4d1d8 100644
--- a/lib/asn1/test/testPrim.erl
+++ b/lib/asn1/test/testPrim.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -37,21 +37,10 @@ compile(Config,Rules,Opt) ->
?line DataDir = ?config(data_dir,Config),
?line OutDir = ?config(priv_dir,Config),
?line true = code:add_patha(?config(priv_dir,Config)),
- case Opt of
- [optimize] ->
- ?line ok = asn1ct:compile(DataDir ++ "Prim",
- [Rules,optimize,{outdir,OutDir}]),
- ?line ok = asn1ct:compile(DataDir ++ "Real",
- [Rules,optimize,{outdir,OutDir}]);
- __ ->
- ?line ok = asn1ct:compile(DataDir ++ "Prim",
- [Rules,{outdir,OutDir}]),
- ?line ok = asn1ct:compile(DataDir ++ "Real",
- [Rules,{outdir,OutDir}])
- end.
-
-
-
+ ?line ok = asn1ct:compile(DataDir ++ "Prim",
+ [Rules,{outdir,OutDir}] ++ Opt),
+ ?line ok = asn1ct:compile(DataDir ++ "Real",
+ [Rules,{outdir,OutDir}] ++ Opt).
bool(Rules) ->
diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl
index f8128702dd..2514c06360 100644
--- a/lib/compiler/src/erl_bifs.erl
+++ b/lib/compiler/src/erl_bifs.erl
@@ -72,7 +72,6 @@ is_pure(erlang, binary_to_list, 1) -> true;
is_pure(erlang, binary_to_list, 3) -> true;
is_pure(erlang, bit_size, 1) -> true;
is_pure(erlang, byte_size, 1) -> true;
-is_pure(erlang, concat_binary, 1) -> true;
is_pure(erlang, element, 2) -> true;
is_pure(erlang, float, 1) -> true;
is_pure(erlang, float_to_list, 1) -> true;
diff --git a/lib/debugger/src/dbg_iload.erl b/lib/debugger/src/dbg_iload.erl
index 2ae0c333da..db5a17ad2e 100644
--- a/lib/debugger/src/dbg_iload.erl
+++ b/lib/debugger/src/dbg_iload.erl
@@ -635,7 +635,6 @@ bif_type(disconnect_node) -> safe;
bif_type(binary_to_list) -> safe;
bif_type(list_to_binary) -> safe;
bif_type(split_binary) -> safe;
-bif_type(concat_binary) -> safe;
bif_type(term_to_atom) -> safe;
bif_type(hash) -> safe;
bif_type(pre_loaded) -> safe;
diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl
index f1be658054..64a695129b 100644
--- a/lib/hipe/cerl/erl_bif_types.erl
+++ b/lib/hipe/cerl/erl_bif_types.erl
@@ -675,8 +675,6 @@ type(erlang, cancel_timer, 1, Xs) ->
type(erlang, check_process_code, 2, Xs) ->
strict(arg_types(erlang, check_process_code, 2), Xs,
fun (_) -> t_boolean() end);
-type(erlang, concat_binary, 1, Xs) ->
- strict(arg_types(erlang, concat_binary, 1), Xs, fun (_) -> t_binary() end);
type(erlang, crc32, 1, Xs) ->
strict(arg_types(erlang, crc32, 1), Xs, fun (_) -> t_crc32() end);
type(erlang, crc32, 2, Xs) ->
@@ -3397,8 +3395,6 @@ arg_types(erlang, cancel_timer, 1) ->
[t_reference()];
arg_types(erlang, check_process_code, 2) ->
[t_pid(), t_atom()];
-arg_types(erlang, concat_binary, 1) ->
- [t_list(t_binary())];
arg_types(erlang, crc32, 1) ->
[t_iodata()];
arg_types(erlang, crc32, 2) ->
diff --git a/lib/kernel/doc/src/gen_sctp.xml b/lib/kernel/doc/src/gen_sctp.xml
index 5ceb82ae41..b761b6bd83 100644
--- a/lib/kernel/doc/src/gen_sctp.xml
+++ b/lib/kernel/doc/src/gen_sctp.xml
@@ -990,7 +990,7 @@
server(IP, Port) when is_tuple(IP) orelse IP == any orelse IP == loopback,
is_integer(Port) -&gt;
- {ok,S} = gen_sctp:open([{ip,IP},{port,Port}],[{recbuf,65536}]),
+ {ok,S} = gen_sctp:open(Port, [{recbuf,65536}, {ip,IP}]),
io:format("Listening on ~w:~w. ~w~n", [IP,Port,S]),
ok = gen_sctp:listen(S, true),
server_loop(S).
diff --git a/lib/sasl/src/release_handler.erl b/lib/sasl/src/release_handler.erl
index eb29787103..ab54b1d00b 100644
--- a/lib/sasl/src/release_handler.erl
+++ b/lib/sasl/src/release_handler.erl
@@ -291,7 +291,8 @@ check_script(Script, LibDirs) ->
release_handler_1:check_script(Script, LibDirs).
%%-----------------------------------------------------------------
-%% eval_script(Script, Apps, LibDirs, Opts) -> {ok, UnPurged} |
+%% eval_script(Script, Apps, LibDirs, NewLibs, Opts) ->
+%% {ok, UnPurged} |
%% restart_new_emulator |
%% {error, Error}
%% {'EXIT', Reason}
@@ -299,9 +300,13 @@ check_script(Script, LibDirs) ->
%% net_kernel:monitor_nodes(true) before calling this function.
%% No! No other process than the release_handler can ever call this
%% function, if sync_nodes is used.
-%%-----------------------------------------------------------------
-eval_script(Script, Apps, LibDirs, Opts) ->
- catch release_handler_1:eval_script(Script, Apps, LibDirs, Opts).
+%%
+%% LibDirs is a list of all applications, while NewLibs is a list of
+%% applications that have changed version between the current and the
+%% new release.
+%% -----------------------------------------------------------------
+eval_script(Script, Apps, LibDirs, NewLibs, Opts) ->
+ catch release_handler_1:eval_script(Script, Apps, LibDirs, NewLibs, Opts).
%%-----------------------------------------------------------------
%% Func: create_RELEASES(Root, RelFile, LibDirs) -> ok | {error, Reason}
@@ -405,6 +410,7 @@ eval_appup_script(App, ToVsn, ToDir, Script) ->
Res = release_handler_1:eval_script(Script,
[], % [AppSpec]
[{App, ToVsn, ToDir}],
+ [{App, ToVsn, ToDir}],
[]), % [Opt]
case Res of
{ok, _Unpurged} ->
@@ -906,7 +912,9 @@ do_install_release(#state{start_prg = StartPrg,
EnvBefore = application_controller:prep_config_change(),
Apps = change_appl_data(RelDir, Release, Masters),
LibDirs = Release#release.libs,
- case eval_script(Script, Apps, LibDirs, Opts) of
+ NewLibs = get_new_libs(LatestRelease#release.libs,
+ Release#release.libs),
+ case eval_script(Script, Apps, LibDirs, NewLibs, Opts) of
{ok, []} ->
application_controller:config_change(EnvBefore),
mon_nodes(false),
@@ -1946,3 +1954,25 @@ safe_write_file_m(File, Data, Masters) ->
filename:basename(File),
filename:basename(Backup)}})
end.
+
+%%-----------------------------------------------------------------
+%% Figure out which applications that have changed version between the
+%% two releases. The paths for these applications must always be
+%% updated, even if the relup script does not load any modules. See
+%% OTP-9402.
+%%
+%% A different situation is when the same application version is used
+%% in old and new release, but the path has changed. This is not
+%% handled here - instead it must be explicitely indicated by the
+%% 'update_paths' option to release_handler:install_release/2 if the
+%% code path shall be updated then.
+%% -----------------------------------------------------------------
+get_new_libs([{App,Vsn,LibDir}|CurrentLibs], NewLibs) ->
+ case lists:keyfind(App,1,NewLibs) of
+ {App,NewVsn,_} = LibInfo when NewVsn =/= Vsn ->
+ [LibInfo | get_new_libs(CurrentLibs,NewLibs)];
+ _ ->
+ get_new_libs(CurrentLibs,NewLibs)
+ end;
+get_new_libs([],_) ->
+ [].
diff --git a/lib/sasl/src/release_handler_1.erl b/lib/sasl/src/release_handler_1.erl
index 8d050fb7b0..ff62f847ac 100644
--- a/lib/sasl/src/release_handler_1.erl
+++ b/lib/sasl/src/release_handler_1.erl
@@ -19,7 +19,7 @@
-module(release_handler_1).
%% External exports
--export([eval_script/3, eval_script/4, check_script/2]).
+-export([eval_script/1, eval_script/5, check_script/2]).
-export([get_current_vsn/1]). %% exported because used in a test case
-record(eval_state, {bins = [], stopped = [], suspended = [], apps = [],
@@ -33,11 +33,11 @@
%% libdirs = [{Lib, LibVsn, LibDir}] - Maps Lib to Vsn and Directory
%% unpurged = [{Mod, soft_purge | brutal_purge}]
%% vsns = [{Mod, OldVsn, NewVsn}] - remember the old vsn of a mod
-%% before it is removed/a new vsn is loaded; the new vsn
+%% before a new vsn is loaded; the new vsn
%% is kept in case of a downgrade, where the code_change
%% function receives the vsn of the module to downgrade
%% *to*.
-%% newlibs = [{Lib, Dir}] - list of all new libs; used to change
+%% newlibs = [{Lib, LibVsn, LibDir}] - list of all new libs; used to change
%% the code path
%% opts = [{Tag, Value}] - list of options
%%-----------------------------------------------------------------
@@ -63,10 +63,11 @@ check_script(Script, LibDirs) ->
{error, {old_processes, Mod}}
end.
-eval_script(Script, Apps, LibDirs) ->
- eval_script(Script, Apps, LibDirs, []).
+%% eval_script/1 - For testing only - no apps added, just testing instructions
+eval_script(Script) ->
+ eval_script(Script, [], [], [], []).
-eval_script(Script, Apps, LibDirs, Opts) ->
+eval_script(Script, Apps, LibDirs, NewLibs, Opts) ->
case catch check_old_processes(Script) of
ok ->
{Before, After} = split_instructions(Script),
@@ -75,6 +76,7 @@ eval_script(Script, Apps, LibDirs, Opts) ->
end,
#eval_state{apps = Apps,
libdirs = LibDirs,
+ newlibs = NewLibs,
opts = Opts},
Before) of
EvalState2 when is_record(EvalState2, eval_state) ->
@@ -214,16 +216,15 @@ check_old_code(Mod) ->
%%-----------------------------------------------------------------
eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) ->
case lists:keysearch(Lib, 1, EvalState#eval_state.libdirs) of
- {value, {Lib, LibVsn, LibDir}} ->
- Ebin = filename:join(LibDir, "ebin"),
+ {value, {Lib, LibVsn, LibDir} = LibInfo} ->
Ext = code:objfile_extension(),
{NewBins, NewVsns} =
lists:foldl(fun(Mod, {Bins, Vsns}) ->
File = lists:concat([Mod, Ext]),
- FName = filename:join(Ebin, File),
+ FName = filename:join([LibDir, "ebin", File]),
case erl_prim_loader:get_file(FName) of
{ok, Bin, FName2} ->
- NVsns = add_new_vsn(Mod, Bin, Vsns),
+ NVsns = add_vsns(Mod, Bin, Vsns),
{[{Mod, Bin, FName2} | Bins],NVsns};
error ->
throw({error, {no_such_file,FName}})
@@ -232,7 +233,7 @@ eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) ->
{EvalState#eval_state.bins,
EvalState#eval_state.vsns},
Modules),
- NewLibs = [{Lib, Ebin} | EvalState#eval_state.newlibs],
+ NewLibs = lists:keystore(Lib,1,EvalState#eval_state.newlibs,LibInfo),
EvalState#eval_state{bins = NewBins,
newlibs = NewLibs,
vsns = NewVsns};
@@ -242,15 +243,14 @@ eval({load_object_code, {Lib, LibVsn, Modules}}, EvalState) ->
eval(point_of_no_return, EvalState) ->
Libs = case get_opt(update_paths, EvalState, false) of
false ->
- EvalState#eval_state.newlibs; % [{Lib, Path}]
+ EvalState#eval_state.newlibs;
true ->
- lists:map(fun({Lib, _LibVsn, LibDir}) ->
- Ebin= filename:join(LibDir,"ebin"),
- {Lib, Ebin}
- end,
- EvalState#eval_state.libdirs)
+ EvalState#eval_state.libdirs
end,
- lists:foreach(fun({Lib, Path}) -> code:replace_path(Lib, Path) end,
+ lists:foreach(fun({Lib, _LibVsn, LibDir}) ->
+ Ebin = filename:join(LibDir,"ebin"),
+ code:replace_path(Lib, Ebin)
+ end,
Libs),
EvalState;
eval({load, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) ->
@@ -258,32 +258,21 @@ eval({load, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) ->
{value, {_Mod, Bin, File}} = lists:keysearch(Mod, 1, Bins),
% load_binary kills all procs running old code
% if soft_purge, we know that there are no such procs now
- Vsns = EvalState#eval_state.vsns,
- NewVsns = add_old_vsn(Mod, Vsns),
code:load_binary(Mod, File, Bin),
% Now, the prev current is old. There might be procs
% running it. Find them.
Unpurged = do_soft_purge(Mod,PostPurgeMethod,EvalState#eval_state.unpurged),
EvalState#eval_state{bins = lists:keydelete(Mod, 1, Bins),
- unpurged = Unpurged,
- vsns = NewVsns};
+ unpurged = Unpurged};
eval({remove, {Mod, _PrePurgeMethod, PostPurgeMethod}}, EvalState) ->
- % purge kills all procs running old code
- % if soft_purge, we know that there are no such procs now
- Vsns = EvalState#eval_state.vsns,
- NewVsns = add_old_vsn(Mod, Vsns),
+ %% purge kills all procs running old code
+ %% if soft_purge, we know that there are no such procs now
code:purge(Mod),
code:delete(Mod),
- % Now, the prev current is old. There might be procs
- % running it. Find them.
- Unpurged =
- case code:soft_purge(Mod) of
- true -> EvalState#eval_state.unpurged;
- false -> [{Mod, PostPurgeMethod} | EvalState#eval_state.unpurged]
- end,
-%% Bins = EvalState#eval_state.bins,
-%% EvalState#eval_state{bins = lists:keydelete(Mod, 1, Bins),
- EvalState#eval_state{unpurged = Unpurged, vsns = NewVsns};
+ %% Now, the prev current is old. There might be procs
+ %% running it. Find them.
+ Unpurged = do_soft_purge(Mod,PostPurgeMethod,EvalState#eval_state.unpurged),
+ EvalState#eval_state{unpurged = Unpurged};
eval({purge, Modules}, EvalState) ->
% Now, if there are any processes still executing old code, OR
% if some new processes started after suspend but before load,
@@ -606,26 +595,20 @@ sync_nodes(Id, Nodes) ->
end,
NNodes).
-add_old_vsn(Mod, Vsns) ->
+add_vsns(Mod, NewBin, Vsns) ->
+ OldVsn = get_current_vsn(Mod),
+ NewVsn = get_vsn(NewBin),
case lists:keysearch(Mod, 1, Vsns) of
- {value, {Mod, undefined, NewVsn}} ->
- OldVsn = get_current_vsn(Mod),
- lists:keyreplace(Mod, 1, Vsns, {Mod, OldVsn, NewVsn});
- {value, {Mod, _OldVsn, _NewVsn}} ->
- Vsns;
+ {value, {Mod, OldVsn0, NewVsn0}} ->
+ lists:keyreplace(Mod, 1, Vsns, {Mod,
+ replace_undefined(OldVsn0,OldVsn),
+ replace_undefined(NewVsn0,NewVsn)});
false ->
- OldVsn = get_current_vsn(Mod),
- [{Mod, OldVsn, undefined} | Vsns]
+ [{Mod, OldVsn, NewVsn} | Vsns]
end.
-add_new_vsn(Mod, Bin, Vsns) ->
- NewVsn = get_vsn(Bin),
- case lists:keysearch(Mod, 1, Vsns) of
- {value, {Mod, OldVsn, undefined}} ->
- lists:keyreplace(Mod, 1, Vsns, {Mod, OldVsn, NewVsn});
- false ->
- [{Mod, undefined, NewVsn} | Vsns]
- end.
+replace_undefined(undefined,Vsn) -> Vsn;
+replace_undefined(Vsn,_) -> Vsn.
%%-----------------------------------------------------------------
%% Func: get_current_vsn/1
@@ -645,7 +628,9 @@ get_current_vsn(Mod) ->
{ok, Bin, _File2} ->
get_vsn(Bin);
error ->
- throw({error, {no_such_file, File}})
+ %% This is the case when a new module is added, there will
+ %% be no current version of it at the time of this call.
+ undefined
end.
%%-----------------------------------------------------------------
diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl
index 16267ba0d4..9c7733b7ec 100644
--- a/lib/sasl/test/release_handler_SUITE.erl
+++ b/lib/sasl/test/release_handler_SUITE.erl
@@ -55,7 +55,7 @@ win32_cases() ->
%% Cases that can be run on all platforms
cases() ->
- [otp_2740, otp_2760, otp_5761, instructions, eval_appup].
+ [otp_2740, otp_2760, otp_5761, otp_9402, otp_9417, instructions, eval_appup].
groups() ->
[{release,[],
@@ -393,7 +393,7 @@ instructions(Conf) when is_list(Conf) ->
{stop, [aa]},
{apply, {?MODULE, no_cc, []}},
{start, [aa]}],
- {ok, _} = release_handler_1:eval_script(S1, [], []),
+ {ok, _} = release_handler_1:eval_script(S1),
case whereis(cc) of
Pid2 when is_pid(Pid2) -> ok;
@@ -403,17 +403,17 @@ instructions(Conf) when is_list(Conf) ->
%% Make bb run old version of b.
S2 = [point_of_no_return,
{remove, {b, soft_purge, soft_purge}}],
- {ok, [{b, soft_purge}]} = release_handler_1:eval_script(S2, [], []),
+ {ok, [{b, soft_purge}]} = release_handler_1:eval_script(S2),
check_bstate("first", [FirstBB]),
false = code:is_loaded(b),
- {error,{old_processes,b}} = release_handler_1:eval_script(S2,[],[]),
+ {error,{old_processes,b}} = release_handler_1:eval_script(S2),
check_bstate("first", [FirstBB]),
%% Let supervisor restart bb with new code
S3 = [point_of_no_return,
{purge, [b]}],
- {ok, []} = release_handler_1:eval_script(S3, [], []),
+ {ok, []} = release_handler_1:eval_script(S3),
ok = wait_for(bb),
check_bstate("second", []),
SecondBB = whereis(bb),
@@ -446,7 +446,7 @@ instructions(Conf) when is_list(Conf) ->
%% Let supervisor restart bb yet another time
S4 = [point_of_no_return,
{remove, {b, brutal_purge, soft_purge}}],
- {ok, HopefullyEmpty} = release_handler_1:eval_script(S4, [], []),
+ {ok, HopefullyEmpty} = release_handler_1:eval_script(S4),
ok = wait_for(bb),
FourthBB = whereis(bb),
@@ -566,8 +566,7 @@ otp_2760(Conf) ->
%% Execute the relup script and check that app1 is unloaded
{ok, [{"after", [{_Rel1Vsn, _Descr, Script}], _}]} =
file:consult(filename:join(Rel2Dir, "relup")),
- {ok, []} = rpc:call(Node, release_handler_1, eval_script,
- [Script, [], []]),
+ {ok, []} = rpc:call(Node, release_handler_1, eval_script, [Script]),
false = rpc:call(Node, code, is_loaded, [app1]),
true = stop_node(Node),
@@ -658,6 +657,126 @@ otp_5761(Conf) when is_list(Conf) ->
true = stop_node(Node),
ok.
+
+%% When a new version of an application is added, but no module is
+%% changed - the path was not updated - i.e. code:priv_dir would point
+%% to the old location.
+otp_9402(Conf) when is_list(Conf) ->
+ %% Set some paths
+ PrivDir = priv_dir(Conf),
+ Dir = filename:join(PrivDir,"otp_9402"),
+ LibDir = filename:join(?config(data_dir, Conf), "lib"),
+
+ %% Create the releases
+ Rel1 = create_and_install_fake_first_release(Dir,
+ [{a,"1.1",LibDir}]),
+ Rel2 = create_fake_upgrade_release(Dir,
+ "2",
+ [{a,"1.2",LibDir}],
+ {Rel1,Rel1,[LibDir]}),
+ Rel1Dir = filename:dirname(Rel1),
+ Rel2Dir = filename:dirname(Rel2),
+
+ %% Start a slave node
+ {ok, Node} = t_start_node(otp_9402, Rel1, filename:join(Rel1Dir,"sys.config")),
+
+ %% Check path
+ Dir1 = filename:join([LibDir, "a-1.1"]),
+ Dir1 = rpc:call(Node, code, lib_dir, [a]),
+ ABeam = rpc:call(Node, code, which, [a]),
+
+ %% Install second release, with no changed modules
+ {ok, RelVsn2} =
+ rpc:call(Node, release_handler, set_unpacked,
+ [Rel2++".rel", [{a,"1.2",LibDir}]]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "relup")]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "sys.config")]),
+
+ {ok, RelVsn1, []} =
+ rpc:call(Node, release_handler, install_release, [RelVsn2]),
+
+ %% Check path
+ Dir2 = filename:join([LibDir, "a-1.2"]),
+ Dir2 = rpc:call(Node, code, lib_dir, [a]),
+ APrivDir2 = rpc:call(Node, code, priv_dir, [a]),
+ true = filelib:is_regular(filename:join(APrivDir2,"file")),
+
+ %% Just to make sure no modules have been re-loaded
+ ABeam = rpc:call(Node, code, which, [a]),
+
+ %% Install RelVsn1 again
+ {ok, _OtherVsn, []} =
+ rpc:call(Node, release_handler, install_release, [RelVsn1]),
+
+ %% Check path
+ Dir1 = rpc:call(Node, code, lib_dir, [a]),
+ APrivDir1 = rpc:call(Node, code, priv_dir, [a]),
+ false = filelib:is_regular(filename:join(APrivDir1,"file")),
+
+ %% Just to make sure no modules have been re-loaded
+ ABeam = rpc:call(Node, code, which, [a]),
+
+ ok.
+
+
+%% When a module is deleted in an appup instruction, the upgrade
+%% failed if the module was not loaded.
+otp_9417(Conf) when is_list(Conf) ->
+ %% Set some paths
+ PrivDir = priv_dir(Conf),
+ Dir = filename:join(PrivDir,"otp_9417"),
+ LibDir = filename:join(?config(data_dir, Conf), "lib"),
+
+ %% Create the releases
+ Rel1 = create_and_install_fake_first_release(Dir,
+ [{b,"1.0",LibDir}]),
+ Rel2 = create_fake_upgrade_release(Dir,
+ "2",
+ [{b,"2.0",LibDir}],
+ {Rel1,Rel1,[LibDir]}),
+ Rel1Dir = filename:dirname(Rel1),
+ Rel2Dir = filename:dirname(Rel2),
+
+ %% Start a slave node
+ {ok, Node} = t_start_node(otp_9417, Rel1, filename:join(Rel1Dir,"sys.config")),
+
+ %% Check paths
+ Dir1 = filename:join([LibDir, "b-1.0"]),
+ Dir1 = rpc:call(Node, code, lib_dir, [b]),
+ BLibBeam = filename:join([Dir1,"ebin","b_lib.beam"]),
+ BLibBeam = rpc:call(Node,code,which,[b_lib]),
+ false = rpc:call(Node,code,is_loaded,[b_lib]),
+ false = rpc:call(Node,code,is_loaded,[b_server]),
+
+ %% Install second release, which removes b_lib module
+ {ok, RelVsn2} =
+ rpc:call(Node, release_handler, set_unpacked,
+ [Rel2++".rel", [{b,"2.0",LibDir}]]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "relup")]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
+ ok = rpc:call(Node, release_handler, install_file,
+ [RelVsn2, filename:join(Rel2Dir, "sys.config")]),
+
+ {ok, _RelVsn1, []} =
+ rpc:call(Node, release_handler, install_release, [RelVsn2]),
+
+ %% Check that the module does no longer exist
+ false = rpc:call(Node, code, is_loaded, [b_lib]),
+ non_existing = rpc:call(Node, code, which, [b_lib]),
+
+ %% And check some paths
+ Dir2 = filename:join([LibDir, "b-2.0"]),
+ Dir2 = rpc:call(Node, code, lib_dir, [b]),
+ BServerBeam = filename:join([Dir2,"ebin","b_server.beam"]),
+ {file,BServerBeam} = rpc:call(Node,code,is_loaded,[b_server]),
+ ok.
+
%% Test upgrade and downgrade of applications
eval_appup(Conf) when is_list(Conf) ->
diff --git a/lib/sasl/test/release_handler_SUITE_data/Makefile.src b/lib/sasl/test/release_handler_SUITE_data/Makefile.src
index 85e25fdc2f..a12e526d2e 100644
--- a/lib/sasl/test/release_handler_SUITE_data/Makefile.src
+++ b/lib/sasl/test/release_handler_SUITE_data/Makefile.src
@@ -1,14 +1,19 @@
EFLAGS=+debug_info
P2B= \
- P2B/a-2.0/ebin/a.beam \
- P2B/a-2.0/ebin/a_sup.beam
+ P2B/a-2.0/ebin/a.@EMULATOR@ \
+ P2B/a-2.0/ebin/a_sup.@EMULATOR@
LIB= \
- lib/a-1.1/ebin/a.beam \
- lib/a-1.1/ebin/a_sup.beam \
- lib/a-1.0/ebin/a.beam \
- lib/a-1.0/ebin/a_sup.beam \
+ lib/a-1.2/ebin/a.@EMULATOR@ \
+ lib/a-1.2/ebin/a_sup.@EMULATOR@ \
+ lib/a-1.1/ebin/a.@EMULATOR@ \
+ lib/a-1.1/ebin/a_sup.@EMULATOR@ \
+ lib/a-1.0/ebin/a.@EMULATOR@ \
+ lib/a-1.0/ebin/a_sup.@EMULATOR@ \
+ lib/b-1.0/ebin/b_server.@EMULATOR@ \
+ lib/b-1.0/ebin/b_lib.@EMULATOR@ \
+ lib/b-2.0/ebin/b_server.@EMULATOR@
APP= \
app1_app2/lib1/app1-1.0/ebin/app1_sup.@EMULATOR@ \
@@ -57,6 +62,21 @@ lib/a-1.1/ebin/a_sup.@EMULATOR@: lib/a-1.1/src/a_sup.erl
erlc $(EFLAGS) -olib/a-1.1/ebin lib/a-1.1/src/a_sup.erl
+lib/a-1.2/ebin/a.@EMULATOR@: lib/a-1.2/src/a.erl
+ erlc $(EFLAGS) -olib/a-1.2/ebin lib/a-1.2/src/a.erl
+lib/a-1.2/ebin/a_sup.@EMULATOR@: lib/a-1.2/src/a_sup.erl
+ erlc $(EFLAGS) -olib/a-1.2/ebin lib/a-1.2/src/a_sup.erl
+
+lib/b-1.0/ebin/b_server.@EMULATOR@: lib/b-1.0/src/b_server.erl
+ erlc $(EFLAGS) -olib/b-1.0/ebin lib/b-1.0/src/b_server.erl
+lib/b-1.0/ebin/b_lib.@EMULATOR@: lib/b-1.0/src/b_lib.erl
+ erlc $(EFLAGS) -olib/b-1.0/ebin lib/b-1.0/src/b_lib.erl
+
+lib/b-2.0/ebin/b_server.@EMULATOR@: lib/b-2.0/src/b_server.erl
+ erlc $(EFLAGS) -olib/b-2.0/ebin lib/b-2.0/src/b_server.erl
+
+
+
app1_app2/lib1/app1-1.0/ebin/app1_sup.@EMULATOR@: app1_app2/lib1/app1-1.0/src/app1_sup.erl
erlc $(EFLAGS) -oapp1_app2/lib1/app1-1.0/ebin app1_app2/lib1/app1-1.0/src/app1_sup.erl
app1_app2/lib1/app1-1.0/ebin/app1_server.@EMULATOR@: app1_app2/lib1/app1-1.0/src/app1_server.erl
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/README b/lib/sasl/test/release_handler_SUITE_data/lib/README
new file mode 100644
index 0000000000..667d21d4cf
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/README
@@ -0,0 +1,16 @@
+a-1.0:
+start version
+
+a-1.1:
+can be upgraded to from a-1.0. Module a has changed
+
+a-1.2:
+can be upgraded to from a-1.1.
+No module have changed, but priv dir is added including one 'file'
+
+b-1.0:
+start version, includes b_lib and b_server
+
+b-2.0:
+can be upgraded to from b-1.0.
+Removes b_lib and updates b_server
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.0/src/a.app b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.app
index e938137f67..b38722f06d 100644
--- a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.0/src/a.app
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.app
@@ -1,7 +1,7 @@
{application, a,
[{description, "A CXC 138 11"},
- {vsn, "1.0"},
- {modules, [{a, 1}, {a_sup,1}]},
+ {vsn, "1.2"},
+ {modules, [{a, 2}, {a_sup,1}]},
{registered, [a_sup]},
{applications, [kernel, stdlib]},
{env, [{key1, val1}]},
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup
new file mode 100644
index 0000000000..3df0546316
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/ebin/a.appup
@@ -0,0 +1,3 @@
+{"1.2",
+ [{"1.1",[]}],
+ [{"1.1",[]}]}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/priv/file
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl
new file mode 100644
index 0000000000..c082ad5339
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a.erl
@@ -0,0 +1,54 @@
+%% ``The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved via the world wide web at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% The Initial Developer of the Original Code is Ericsson Utvecklings AB.
+%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
+%% AB. All Rights Reserved.''
+%%
+%% $Id$
+%%
+-module(a).
+
+
+-behaviour(gen_server).
+
+%% External exports
+-export([start_link/0, a/0, b/0]).
+%% Internal exports
+-export([init/1, handle_call/3, handle_info/2, terminate/2, code_change/3]).
+
+start_link() -> gen_server:start_link({local, aa}, a, [], []).
+
+a() -> gen_server:call(aa, a).
+b() -> gen_server:call(aa, b).
+
+%%-----------------------------------------------------------------
+%% Callback functions from gen_server
+%%-----------------------------------------------------------------
+init([]) ->
+ process_flag(trap_exit, true),
+ {ok, {state, bval}}.
+
+handle_call(a, _From, State) ->
+ X = application:get_all_env(a),
+ {reply, X, State};
+
+handle_call(b, _From, State) ->
+ {reply, {ok, element(2, State)}, State}.
+
+handle_info(_, State) ->
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(1, Extra, State) ->
+ {ok, {state, bval}}.
diff --git a/lib/asn1/src/asn1_sup.erl b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a_sup.erl
index a241dec6f4..a141c1767b 100644
--- a/lib/asn1/src/asn1_sup.erl
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/a-1.2/src/a_sup.erl
@@ -3,35 +3,35 @@
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved via the world wide web at http://www.erlang.org/.
-%%
+%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
-%%
+%%
%% The Initial Developer of the Original Code is Ericsson Utvecklings AB.
%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
%% AB. All Rights Reserved.''
-%%
+%%
%% $Id$
%%
+-module(a_sup).
-%% Purpose: Main supervisor in asn1 application.
-
--module(asn1_sup).
-behaviour(supervisor).
--export([start_link/0, init/1]).
+%% External exports
+-export([start/2]).
-start_link() ->
- supervisor:start_link({local, asn1_sup}, asn1_sup, []).
+%% Internal exports
+-export([init/1]).
+start(_, _) ->
+ supervisor:start_link({local, a_sup}, a_sup, []).
-%% init([])
-%% Returns: {ok, {SupFlags, [ChildSpec]}}
-%%
init([]) ->
- Child = {asn1_server, {asn1_server, start_link, []},
- permanent, 2000, worker, [asn1_server]},
- {ok, {{one_for_all, 10, 3600}, [Child]}}.
+ SupFlags = {one_for_one, 4, 3600},
+ Config = {a,
+ {a, start_link, []},
+ permanent, 2000, worker, [a]},
+ {ok, {SupFlags, [Config]}}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app
new file mode 100644
index 0000000000..00347b2754
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/ebin/b.app
@@ -0,0 +1,7 @@
+%% -*- erlang -*-
+{application, b,
+ [{description, "B CXC 138 12"},
+ {vsn, "1.0"},
+ {modules, [{b_server, 1},{b_lib, 1}]},
+ {registered, [b_server]},
+ {applications, [kernel, stdlib]}]}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl
new file mode 100644
index 0000000000..7e8a308a5e
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_lib.erl
@@ -0,0 +1,3 @@
+-module(b_lib).
+-compile(export_all).
+foo() -> ok.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl
new file mode 100644
index 0000000000..e1a80a076f
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-1.0/src/b_server.erl
@@ -0,0 +1,37 @@
+-module(b_server).
+
+-behaviour(gen_server).
+
+%% API
+-export([start_link/0]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+ terminate/2, code_change/3]).
+
+-define(SERVER, ?MODULE).
+
+-record(state, {}).
+
+start_link() ->
+ gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+
+init([]) ->
+ {ok, #state{}}.
+
+handle_call(_Request, _From, State) ->
+ Reply = ok,
+ {reply, Reply, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info(_Info, State) ->
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(OldVsn, State, Extra) ->
+ file:write_file("/tmp/b_server",io_lib:format("~p~n",[{"1.0",OldVsn,Extra}])),
+ {ok, State}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app
new file mode 100644
index 0000000000..73c8e42b32
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.app
@@ -0,0 +1,7 @@
+%% -*- erlang -*-
+{application, b,
+ [{description, "B CXC 138 12"},
+ {vsn, "2.0"},
+ {modules, [{b_server, 1}]},
+ {registered, [b_server]},
+ {applications, [kernel, stdlib]}]}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup
new file mode 100644
index 0000000000..d261a37732
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/ebin/b.appup
@@ -0,0 +1,3 @@
+{"2.0",
+ [{"1.0",[{delete_module,b_lib}, {update,b_server,{advanced,[]}}]}],
+ [{"1.0",[{add_module,b_lib},{update,b_server,{advanced,[]}}]}]}.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl
new file mode 100644
index 0000000000..7e8a308a5e
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_lib.erl
@@ -0,0 +1,3 @@
+-module(b_lib).
+-compile(export_all).
+foo() -> ok.
diff --git a/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl
new file mode 100644
index 0000000000..f8bfbdaff7
--- /dev/null
+++ b/lib/sasl/test/release_handler_SUITE_data/lib/b-2.0/src/b_server.erl
@@ -0,0 +1,37 @@
+-module(b_server).
+
+-behaviour(gen_server).
+
+%% API
+-export([start_link/0]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+ terminate/2, code_change/3]).
+
+-define(SERVER, ?MODULE).
+
+-record(state, {}).
+
+start_link() ->
+ gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+
+init([]) ->
+ {ok, #state{}}.
+
+handle_call(_Request, _From, State) ->
+ Reply = ok,
+ {reply, Reply, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info(_Info, State) ->
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(OldVsn, State, Extra) ->
+ file:write_file("/tmp/b_server",io_lib:format("~p~n",[{"2.0",OldVsn,Extra}])),
+ {ok, State}.
diff --git a/lib/snmp/Makefile b/lib/snmp/Makefile
index 20e3d4692a..c55eff04c6 100644
--- a/lib/snmp/Makefile
+++ b/lib/snmp/Makefile
@@ -67,7 +67,7 @@ do_configure: configure
configure: configure.in
autoconf
-.PHONY: info
+.PHONY: info gclean
info:
@echo "OS: $(OS)"
@@ -76,6 +76,9 @@ info:
@echo "SNMP_VSN: $(SNMP_VSN)"
@echo "APP_VSN: $(APP_VSN)"
+gclean:
+ git clean -fXd
+
# ----------------------------------------------------
# Application (source) release targets
diff --git a/lib/snmp/doc/src/Makefile b/lib/snmp/doc/src/Makefile
index 35ed63e103..aa9431477c 100644
--- a/lib/snmp/doc/src/Makefile
+++ b/lib/snmp/doc/src/Makefile
@@ -152,6 +152,7 @@ $(TOP_PDF_FILE): $(XML_FILES)
pdf: $(TOP_PDF_FILE)
html: gifs $(HTML_REF_MAN_FILE)
+html2: html $(INDEX_TARGET)
clean clean_docs: clean_html clean_man clean_pdf
rm -f errs core *~
@@ -228,6 +229,7 @@ clean_man:
clean_html:
@echo "cleaning html:"
rm -rf $(HTMLDIR)/*
+ rm -f $(INDEX_TARGET)
$(MAN7DIR)/%.7: $(MIBSDIR)/%.mib
@echo "processing $*"
@@ -286,7 +288,7 @@ release_docs_spec: docs
$(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR)
$(INSTALL_DIR) $(RELEASE_PATH)/man/man1
$(INSTALL_DATA) $(MAN1_FILES) $(RELEASE_PATH)/man/man1
- $(INSTALL_DIR) $(RELEASE_PATH)/man/man
+ $(INSTALL_DIR) $(RELEASE_PATH)/man/man3
$(INSTALL_DATA) $(MAN3_FILES) $(RELEASE_PATH)/man/man3
$(INSTALL_DIR) $(RELEASE_PATH)/man/man6
$(INSTALL_DATA) $(MAN6_FILES) $(RELEASE_PATH)/man/man6
diff --git a/lib/snmp/doc/src/notes.xml b/lib/snmp/doc/src/notes.xml
index 6a20d8ee3a..4178192120 100644
--- a/lib/snmp/doc/src/notes.xml
+++ b/lib/snmp/doc/src/notes.xml
@@ -33,6 +33,136 @@
</header>
<section>
+ <title>SNMP Development Toolkit 4.21</title>
+ <p>Version 4.21 supports code replacement in runtime from/to
+ version 4.20.1, 4.20 and 4.19. </p>
+
+ <section>
+ <title>Improvements and new features</title>
+<!--
+ <p>-</p>
+-->
+ <list type="bulleted">
+ <item>
+ <p>[manager] There was no way to specify transport domain.
+ The transport domains was assumed to be IPv4 (transportDomainUdpIpv4).
+ This has now been changed so that it can also be IPv6
+ (transportDomainUdpIpv6).
+ To facilitate this, the transport domain, <c>tdomain</c>,
+ is now a (new) valid option when
+ <seealso marker="snmpm#register_agent">registering</seealso>
+ a new agent (and
+ <seealso marker="snmpm#update_agent_info">updating</seealso>
+ agent info). </p>
+ <p>This also mean that the transport behaviour has changed. </p>
+ <p>Own Id: OTP-9305</p>
+ <p>Aux Id: Seq 11847</p>
+ </item>
+
+ <item>
+ <p>[compiler] Added the option
+ <seealso marker="snmpc#compile">warnings_as_errors</seealso>
+ (for the SNMP MIB compiler (escript) frontend, the option
+ <seealso marker="snmpc(command)#option_wae">--wae</seealso> is used)
+ which specifies whether warnings should be treated as errors. </p>
+ <p>Tuncer Ayaz</p>
+ <p>Own Id: OTP-9437</p>
+ </item>
+ </list>
+
+ </section>
+
+ <section>
+ <title>Fixed Bugs and Malfunctions</title>
+<!--
+ <p>-</p>
+-->
+
+ <list type="bulleted">
+ <item>
+ <p>The snmp config tool could not handle (manager) audit trail config
+ because the option seqno was not handled. </p>
+ <p>Own Id: OTP-9354</p>
+ </item>
+
+ <item>
+ <p>[agent] The SNMP ACM cache was not properly updated when
+ changes where made to the VACM security-to-group, access and
+ view-tree-family tables. </p>
+ <p>Own Id: OTP-9367</p>
+ <p>Aux Id: Seq 11858</p>
+ </item>
+
+ <item>
+ <p>Fixed install directory typo for man3. </p>
+ <p>Peter Lemenkov</p>
+ <p>Hans Ulrich Niedermann</p>
+ <p>Own Id: OTP-9442</p>
+ </item>
+
+ </list>
+ </section>
+
+
+ <section>
+ <title>Incompatibilities</title>
+ <p>-</p>
+ </section>
+
+ </section> <!-- 4.21 -->
+
+
+ <section>
+ <title>SNMP Development Toolkit 4.20.1</title>
+ <p>Version 4.20.1 supports code replacement in runtime from/to
+ version 4.20, 4.19 and 4.18.</p>
+
+ <section>
+ <title>Improvements and new features</title>
+ <p>-</p>
+<!--
+ <list type="bulleted">
+ <item>
+ <p>Added type specs for functions that do not return. </p>
+ <p>Kostis Sagonas</p>
+ <p>Own Id: OTP-9208</p>
+ </item>
+ </list>
+-->
+ </section>
+
+ <section>
+ <title>Fixed Bugs and Malfunctions</title>
+<!--
+ <p>-</p>
+-->
+ <list type="bulleted">
+ <item>
+ <p>[agent] Did not handle transport domains properly in some cases,
+ for instance trap sending. </p>
+ <p>Own Id: OTP-9400</p>
+ </item>
+
+ <item>
+ <p>[agent] Wrong default transport domain, snmpUDPDomain, instead
+ of transportDomainUdpIpv4. </p>
+ <p>Own Id: OTP-9425</p>
+ <p>Aux Id: Seq 11874</p>
+ </item>
+
+ </list>
+ </section>
+
+
+ <section>
+ <title>Incompatibilities</title>
+ <p>-</p>
+ </section>
+
+ </section> <!-- 4.20.1 -->
+
+
+ <section>
<title>SNMP Development Toolkit 4.20</title>
<p>Version 4.20 supports code replacement in runtime from/to
version 4.19 and 4.18.</p>
diff --git a/lib/snmp/doc/src/snmpc.xml b/lib/snmp/doc/src/snmpc.xml
index 771629492d..61d19251c5 100644
--- a/lib/snmp/doc/src/snmpc.xml
+++ b/lib/snmp/doc/src/snmpc.xml
@@ -48,7 +48,11 @@
<type>
<v>File = string()</v>
<v>Options = [opt()]</v>
- <v>opt() = db() | relaxed_row_name_assign_check() | deprecated() | description() | reference() | group_check() | i() | il() | imports() | module() | module_identity() | module_compliance() | agent_capabilities() | outdir() | no_defs() | verbosity() | warnings()</v>
+ <v>opt() = db() | relaxed_row_name_assign_check() | deprecated() |
+ description() | reference() | group_check() | i() | il() |
+ imports() | module() | module_identity() | module_compliance() |
+ agent_capabilities() | outdir() | no_defs() | verbosity() |
+ warnings() | warnings_as_errors()</v>
<v>db() = {db, volatile|persistent|mnesia}</v>
<v>deprecated() = {deprecated, bool()}</v>
<v>relaxed_row_name_assign_check() = relaxed_row_name_assign_check</v>
@@ -66,6 +70,7 @@
<v>outdir() = {outdir, dir()}</v>
<v>verbosity() = {verbosity, silence|warning|info|log|debug|trace}</v>
<v>warnings() = {warnings, bool()}</v>
+ <v>warnings_as_errors() = warnings_as_errors</v>
<v>dir() = string()</v>
<v>BinFileName = string()</v>
</type>
@@ -200,11 +205,17 @@
<item>
<p>The option <c>warnings</c> specifies whether warning
- messages should be shown. </p>
+ messages should be shown. </p>
<p>Default is <c>true</c>. </p>
</item>
+ <item>
+ <p>The option <c>warnings_as_errors</c>, if present, specifies
+ whether warnings should be treated as errors.</p>
+ </item>
+
</list>
+
<p>The MIB compiler understands both SMIv1 and SMIv2 MIBs. It
uses the <c>MODULE-IDENTITY</c> statement to determine if the MIB is
version 1 or 2.
diff --git a/lib/snmp/doc/src/snmpc_cmd.xml b/lib/snmp/doc/src/snmpc_cmd.xml
index 9358382a10..72116f8981 100644
--- a/lib/snmp/doc/src/snmpc_cmd.xml
+++ b/lib/snmp/doc/src/snmpc_cmd.xml
@@ -50,6 +50,8 @@
with definitions of Erlang constants for the objects in
the MIB, see
<seealso marker="snmpc#mib_to_hrl">mib_to_hrl/1</seealso>. </p>
+
+ <marker id="options"></marker>
</desc>
</func>
</funcs>
@@ -58,15 +60,18 @@
<title>Compiler options</title>
<p>The following options are supported (note that most of these relate
to the compilation of the MIB file):</p>
+ <marker id="option_help"></marker>
<taglist>
<tag>--help</tag>
<item>
<p>Prints help info.</p>
+ <marker id="option_version"></marker>
</item>
<tag>--version</tag>
<item>
<p>Prints application and mib format version.</p>
+ <marker id="option_verbosity"></marker>
</item>
<tag>--verbosity <em>verbosity</em></tag>
@@ -74,11 +79,20 @@
<p>Print debug info. </p>
<p><c>verbosity</c> = <c>trace</c> | <c>debug</c> | <c>log</c> | <c>info</c> | <c>silence</c></p>
<p>Defaults to <c>silence</c>.</p>
+ <marker id="option_warnings"></marker>
</item>
<tag>--warnings</tag>
<item>
<p>Print warning messages. </p>
+ <marker id="option_wae"></marker>
+ </item>
+
+ <tag>--wae</tag>
+ <item>
+ <p>Warnings as errors.
+ Indicates that warnings shall be treated as errors. </p>
+ <marker id="option_odir"></marker>
</item>
<tag>--o <em>directory</em></tag>
@@ -86,6 +100,7 @@
<p>The directory where the compiler should place the output files.
If not specified, output files will be placed in the current working
directory.</p>
+ <marker id="option_idir"></marker>
</item>
<tag>--i <em>Directory</em></tag>
@@ -94,6 +109,7 @@
By default, the current working directory is always included. </p>
<p>This option can be present several times, each time specifying
<em>one</em> path. </p>
+ <marker id="option_ildir"></marker>
</item>
<tag>--il <em>Directory</em></tag>
@@ -106,6 +122,7 @@
the current version may be in the system). The current directory
and the "snmp-home"/priv/mibs/ are always listed last in the
include path. </p>
+ <marker id="option_sgc"></marker>
</item>
<tag>--sgc</tag>
@@ -114,42 +131,50 @@
group check of the mib compiler.
That is, should the OBJECT-GROUP and the NOTIFICATION-GROUP
macro(s) be checked for correctness or not. </p>
+ <marker id="option_dep"></marker>
</item>
<tag>--dep</tag>
<item>
<p>Keep deprecated definition(s).
If not specified the compiler will ignore deprecated definitions. </p>
+ <marker id="option_desc"></marker>
</item>
<tag>--desc</tag>
<item>
<p>The DESCRIPTION field will be included. </p>
+ <marker id="option_ref"></marker>
</item>
<tag>--ref</tag>
<item>
<p>The REFERENCE field will be included. </p>
+ <marker id="option_imp"></marker>
</item>
<tag>--imp</tag>
<item>
<p>The IMPORTS field will be included. </p>
+ <marker id="option_mi"></marker>
</item>
<tag>--mi</tag>
<item>
<p>The MODULE-IDENTITY field will be included. </p>
+ <marker id="option_mc"></marker>
</item>
<tag>--mc</tag>
<item>
<p>The MODULE-COMPLIANCE field will be included. </p>
+ <marker id="option_ac"></marker>
</item>
<tag>--ac</tag>
<item>
<p>The AGENT-CAPABILITIES field will be included. </p>
+ <marker id="option_mod"></marker>
</item>
<tag>--mod <em>module</em></tag>
@@ -157,6 +182,7 @@
<p>The module which implements all the instrumentation functions. </p>
<p>The name of all instrumentation functions must be the
same as the corresponding managed object it implements. </p>
+ <marker id="option_nd"></marker>
</item>
<tag>--nd</tag>
@@ -165,6 +191,7 @@
used if a managed object have no instrumentation function.
Instead this will be reported as an error, and the compilation
aborts. </p>
+ <marker id="option_rrnac"></marker>
</item>
<tag>--rrnac</tag>
@@ -176,6 +203,7 @@
This means that the error will be converted to a warning. </p>
<p>By default it is not included, but if this option is present
it will be. </p>
+ <marker id="see_also"></marker>
</item>
</taglist>
diff --git a/lib/snmp/doc/src/snmpm.xml b/lib/snmp/doc/src/snmpm.xml
index b527d171b0..c36a1b2a24 100644
--- a/lib/snmp/doc/src/snmpm.xml
+++ b/lib/snmp/doc/src/snmpm.xml
@@ -283,27 +283,27 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv
<v>TargetName = target_name()</v>
<v>Config = [agent_config()]</v>
<v>agent_config() = {Item, Val}</v>
- <v>Item = engine_id | address | port | community | timeout | max_message_size | version | sec_model | sec_name | sec_level</v>
+ <v>Item = engine_id | address | port | community | timeout | max_message_size | version | sec_model | sec_name | sec_level | tdomain</v>
<v>Val = term()</v>
<v>Reason = term()</v>
</type>
<desc>
<p>Explicitly instruct the manager to handle this agent, with
- <c>UserId</c> as the responsible user. </p>
- <p>Called to instruct the manager that this agent
- shall be handled. This function is used when
- the user knows in advance which agents the
- manager shall handle.
- Note that there is an alternate way to do the same thing:
- Add the agent to the manager config files (see
- <seealso marker="snmp_manager_config_files#agents">agents.conf</seealso>).</p>
+ <c>UserId</c> as the responsible user. </p>
+ <p>Called to instruct the manager that this agent shall be handled.
+ This function is used when the user knows in advance which agents
+ the manager shall handle.
+ Note that there is an alternate way to do the same thing:
+ Add the agent to the manager config files (see
+ <seealso marker="snmp_manager_config_files#agents">agents.conf</seealso>).</p>
<p><c>TargetName</c> is a non-empty string,
- uniquely identifying the agent. </p>
- <p>The type of <c>Val</c> depends on <c>Item</c>: </p>
+ uniquely identifying the agent. </p>
+ <p>The type of <c>Val</c> depends on <c>Item</c>: </p>
<code type="none"><![CDATA[
[mandatory] engine_id = string()
[mandatory] address = ip_address()
[optional] port = integer()
+[optional] tdomain = transportDomainUdpIpv4 | transportDomainUdpIpv6
[optional] community = string()
[optional] timeout = integer() | snmp_timer()
[optional] max_message_size = integer()
@@ -312,7 +312,9 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv
[optional] sec_name = string()
[optional] sec_level = noAuthNoPriv | authNoPriv | authPriv
]]></code>
- <p>Note that if no <c>Port</c> is given, the default value is used.</p>
+ <p>Note that if no <c>tdomain</c> is given, the default value,
+ <c>transportDomainUdpIpv4</c>, is used.</p>
+ <p>Note that if no <c>port</c> is given, the default value is used.</p>
<marker id="unregister_agent"></marker>
</desc>
@@ -348,17 +350,25 @@ sec_level = noAuthNoPriv | authNoPriv | authPriv
</func>
<func>
+ <name>update_agent_info(UserId, TargetName, Info) -> ok | {error, Reason}</name>
<name>update_agent_info(UserId, TargetName, Item, Val) -> ok | {error, Reason}</name>
<fsummary>Update agent config</fsummary>
<type>
<v>UserId = term()</v>
<v>TargetName = target_name()</v>
- <v>Item = atom()</v>
- <v>Val = term()</v>
+ <v>Info = [{item(), item_value()}]</v>
+ <v>Item = item()</v>
+ <v>item() = atom()</v>
+ <v>Val = item_value()</v>
+ <v>item_value() = term()</v>
<v>Reason = term()</v>
</type>
<desc>
- <p>Update agent config.</p>
+ <p>Update agent config. The function <c>update_agent_info/3</c>
+ should be used when several values needs to be updated atomically. </p>
+ <p>See function
+ <seealso marker="#register_agent">register_agent</seealso>)
+ for more info about what kind of items are allowed. </p>
<marker id="which_agents"></marker>
</desc>
diff --git a/lib/snmp/src/agent/snmp_view_based_acm_mib.erl b/lib/snmp/src/agent/snmp_view_based_acm_mib.erl
index 28469a7b4e..37f6dd3f26 100644
--- a/lib/snmp/src/agent/snmp_view_based_acm_mib.erl
+++ b/lib/snmp/src/agent/snmp_view_based_acm_mib.erl
@@ -247,6 +247,7 @@ add_sec2group(SecModel, SecName, GroupName) ->
Key = [Key1, length(Key2) | Key2],
case table_cre_row(vacmSecurityToGroupTable, Key, Row) of
true ->
+ snmpa_agent:invalidate_ca_cache(),
{ok, Key};
false ->
{error, create_failed}
@@ -260,6 +261,7 @@ add_sec2group(SecModel, SecName, GroupName) ->
delete_sec2group(Key) ->
case table_del_row(vacmSecurityToGroupTable, Key) of
true ->
+ snmpa_agent:invalidate_ca_cache(),
ok;
false ->
{error, delete_failed}
@@ -279,6 +281,7 @@ add_access(GroupName, Prefix, SecModel, SecLevel, Match, RV, WV, NV) ->
Key3 = [SM, SL],
Key = Key1 ++ Key2 ++ Key3,
snmpa_vacm:insert([{Key, Row}], false),
+ snmpa_agent:invalidate_ca_cache(),
{ok, Key};
{error, Reason} ->
{error, Reason};
@@ -287,6 +290,7 @@ add_access(GroupName, Prefix, SecModel, SecLevel, Match, RV, WV, NV) ->
end.
delete_access(Key) ->
+ snmpa_agent:invalidate_ca_cache(),
snmpa_vacm:delete(Key).
@@ -299,6 +303,7 @@ add_view_tree_fam(ViewIndex, SubTree, Status, Mask) ->
Key = [length(Key1) | Key1] ++ [length(Key2) | Key2],
case table_cre_row(vacmViewTreeFamilyTable, Key, Row) of
true ->
+ snmpa_agent:invalidate_ca_cache(),
{ok, Key};
false ->
{error, create_failed}
@@ -312,6 +317,7 @@ add_view_tree_fam(ViewIndex, SubTree, Status, Mask) ->
delete_view_tree_fam(Key) ->
case table_del_row(vacmViewTreeFamilyTable, Key) of
true ->
+ snmpa_agent:invalidate_ca_cache(),
ok;
false ->
{error, delete_failed}
diff --git a/lib/snmp/src/agent/snmpa_agent.erl b/lib/snmp/src/agent/snmpa_agent.erl
index 82a7ec647b..6322f0f21d 100644
--- a/lib/snmp/src/agent/snmpa_agent.erl
+++ b/lib/snmp/src/agent/snmpa_agent.erl
@@ -1626,7 +1626,7 @@ invalidate_ca_cache() ->
MasterAgent ! invalidate_ca_cache;
false ->
%% This is running on a sub-agent node,
- %% so sent skip it
+ %% so skip it
ok
end;
_ -> % Not on this node
diff --git a/lib/snmp/src/agent/snmpa_conf.erl b/lib/snmp/src/agent/snmpa_conf.erl
index 4b88eb69f7..c17a6abbd7 100644
--- a/lib/snmp/src/agent/snmpa_conf.erl
+++ b/lib/snmp/src/agent/snmpa_conf.erl
@@ -424,7 +424,8 @@ target_addr_entry(Name,
EngineId,
TMask) ->
target_addr_entry(Name, Ip, 162, TagList,
- ParamsName, EngineId, TMask, 2048).
+ ParamsName, EngineId,
+ TMask, 2048).
target_addr_entry(Name,
Ip,
@@ -435,7 +436,8 @@ target_addr_entry(Name,
TMask,
MaxMessageSize) ->
target_addr_entry(Name, Ip, Udp, 1500, 3, TagList,
- ParamsName, EngineId, TMask, MaxMessageSize).
+ ParamsName, EngineId,
+ TMask, MaxMessageSize).
target_addr_entry(Name,
Ip,
@@ -448,7 +450,8 @@ target_addr_entry(Name,
TMask,
MaxMessageSize) ->
target_addr_entry(Name, snmp_target_mib:default_domain(), Ip, Udp,
- Timeout, RetryCount, TagList, ParamsName,
+ Timeout, RetryCount, TagList,
+ ParamsName, EngineId,
TMask, MaxMessageSize).
target_addr_entry(Name,
diff --git a/lib/snmp/src/agent/snmpa_mpd.erl b/lib/snmp/src/agent/snmpa_mpd.erl
index 14f62b12f3..4f50b1a674 100644
--- a/lib/snmp/src/agent/snmpa_mpd.erl
+++ b/lib/snmp/src/agent/snmpa_mpd.erl
@@ -32,6 +32,7 @@
-include("SNMP-MPD-MIB.hrl").
-include("SNMPv2-TM.hrl").
-include("SNMP-FRAMEWORK-MIB.hrl").
+-include("TRANSPORT-ADDRESS-MIB.hrl").
-define(VMODULE,"MPD").
-include("snmp_verbosity.hrl").
@@ -981,12 +982,15 @@ generate_discovery_msg2(NoteStore, Pdu,
discovery_note_timeout(Timeout) ->
(Timeout div 100) + 1.
-generate_discovery_msg(NoteStore, {?snmpUDPDomain, [A,B,C,D,U1,U2]},
+generate_discovery_msg(NoteStore, {TDomain, TAddress},
Pdu, ScopedPduBytes,
ContextEngineID, ManagerEngineID,
SecModel, SecName, SecLevelFlag,
InitialUserName,
ContextName, Timeout) ->
+
+ {ok, {_Domain, Address}} = transform_taddr(TDomain, TAddress),
+
%% 7.1.7
?vdebug("generate_discovery_msg -> 7.1.7 (~w)", [ManagerEngineID]),
MsgID = generate_msg_id(),
@@ -1027,7 +1031,7 @@ generate_discovery_msg(NoteStore, {?snmpUDPDomain, [A,B,C,D,U1,U2]},
%% Log(Packet),
inc_snmp_out_vars(Pdu),
?vdebug("generate_discovery_msg -> done", []),
- {Packet, {{A,B,C,D}, U1 bsl 8 + U2}};
+ {Packet, Address};
Error ->
throw(Error)
@@ -1057,6 +1061,34 @@ generate_sec_discovery_msg(Message, SecModule,
end.
+transform_taddr(?snmpUDPDomain, TAddress) ->
+ transform_taddr(?transportDomainUdpIpv4, TAddress);
+transform_taddr(?transportDomainUdpIpv4, [A, B, C, D, P1, P2]) ->
+ Domain = transportDomainUdpIpv4,
+ Addr = {A,B,C,D},
+ Port = P1 bsl 8 + P2,
+ Address = {Addr, Port},
+ {ok, {Domain, Address}};
+transform_taddr(?transportDomainUdpIpv4, BadAddr) ->
+ {error, {bad_transportDomainUdpIpv4_address, BadAddr}};
+transform_taddr(?transportDomainUdpIpv6,
+ [A1, A2, A3, A4, A5, A6, A7, A8, P1, P2]) ->
+ Domain = transportDomainUdpIpv6,
+ Addr = {A1, A2, A3, A4, A5, A6, A7, A8},
+ Port = P1 bsl 8 + P2,
+ Address = {Addr, Port},
+ {ok, {Domain, Address}};
+transform_taddr(?transportDomainUdpIpv6, BadAddr) ->
+ {error, {bad_transportDomainUdpIpv6_address, BadAddr}};
+transform_taddr(BadTDomain, TAddress) ->
+ case lists:member(BadTDomain, snmp_conf:all_tdomains()) of
+ true ->
+ {error, {unsupported_tdomain, BadTDomain, TAddress}};
+ false ->
+ {error, {unknown_tdomain, BadTDomain, TAddress}}
+ end.
+
+
process_taddrs(Dests) ->
?vtrace("process_taddrs -> entry with"
"~n Dests: ~p", [Dests]),
@@ -1066,46 +1098,44 @@ process_taddrs([], Acc) ->
?vtrace("process_taddrs -> entry when done with"
"~n Acc: ~p", [Acc]),
lists:reverse(Acc);
-
+
%% v3
-process_taddrs([{{?snmpUDPDomain, [A,B,C,D,U1,U2]}, SecData} | T], Acc) ->
+process_taddrs([{{TDomain, TAddress}, SecData} | T], Acc) ->
?vtrace("process_taddrs -> entry when v3 with"
- "~n A: ~p"
- "~n B: ~p"
- "~n C: ~p"
- "~n D: ~p"
- "~n U1: ~p"
- "~n U2: ~p"
- "~n SecData: ~p", [A, B, C, D, U1, U2, SecData]),
- Entry = {{snmpUDPDomain, {{A,B,C,D}, U1 bsl 8 + U2}}, SecData},
- process_taddrs(T, [Entry | Acc]);
-%% Bad v3
-process_taddrs([{{TDomain, TAddr}, _SecData} | T], Acc) ->
- ?vtrace("process_taddrs -> entry when bad v3 with"
- "~n TDomain: ~p"
- "~n TAddr: ~p", [TDomain, TAddr]),
- user_err("Bad TDomain/TAddr: ~w/~w", [TDomain, TAddr]),
- process_taddrs(T, Acc);
+ "~n TDomain: ~p"
+ "~n TAddress: ~p"
+ "~n SecData: ~p", [TDomain, TAddress, SecData]),
+ case transform_taddr(TDomain, TAddress) of
+ {ok, DestAddr} ->
+ ?vtrace("process_taddrs -> transformed: "
+ "~n DestAddr: ~p", [DestAddr]),
+ Entry = {DestAddr, SecData},
+ process_taddrs(T, [Entry | Acc]);
+ {error, Reason} ->
+ ?vinfo("Failed transforming v3 domain and address"
+ "~n Reason: ~p", [Reason]),
+ user_err("Bad TDomain/TAddress: ~w/~w", [TDomain, TAddress]),
+ process_taddrs(T, Acc)
+ end;
%% v1 & v2
-process_taddrs([{?snmpUDPDomain, [A,B,C,D,U1,U2]} | T], Acc) ->
+process_taddrs([{TDomain, TAddress} | T], Acc) ->
?vtrace("process_taddrs -> entry when v1/v2 with"
- "~n A: ~p"
- "~n B: ~p"
- "~n C: ~p"
- "~n D: ~p"
- "~n U1: ~p"
- "~n U2: ~p", [A, B, C, D, U1, U2]),
- Entry = {snmpUDPDomain, {{A,B,C,D}, U1 bsl 8 + U2}},
- process_taddrs(T, [Entry | Acc]);
-%% Bad v1 or v2
-process_taddrs([{TDomain, TAddr} | T], Acc) ->
- ?vtrace("process_taddrs -> entry when bad v1/v2 with"
- "~n TDomain: ~p"
- "~n TAddr: ~p", [TDomain, TAddr]),
- user_err("Bad TDomain/TAddr: ~w/~w", [TDomain, TAddr]),
- process_taddrs(T, Acc);
+ "~n TDomain: ~p"
+ "~n TAddress: ~p", [TDomain, TAddress]),
+ case transform_taddr(TDomain, TAddress) of
+ {ok, DestAddr} ->
+ ?vtrace("process_taddrs -> transformed: "
+ "~n DestAddr: ~p", [DestAddr]),
+ Entry = DestAddr,
+ process_taddrs(T, [Entry | Acc]);
+ {error, Reason} ->
+ ?vinfo("Failed transforming v1/v2 domain and address: "
+ "~n Reason: ~p", [Reason]),
+ user_err("Bad TDomain/TAddress: ~w/~w", [TDomain, TAddress]),
+ process_taddrs(T, Acc)
+ end;
process_taddrs(Crap, Acc) ->
- throw({error, {taddrs_crap, Crap, Acc}}).
+ throw({error, {bad_taddrs, Crap, Acc}}).
mk_v1_v2_packet_list(To, Packet, Len, Pdu) ->
diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src
index 5deb40be0f..8e1855b4df 100644
--- a/lib/snmp/src/app/snmp.appup.src
+++ b/lib/snmp/src/app/snmp.appup.src
@@ -22,82 +22,82 @@
%% ----- U p g r a d e -------------------------------------------------------
[
+ {"4.20.1",
+ [
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
+ {load_module, snmp_conf, soft_purge, soft_purge, []},
+ {load_module, snmp_config, soft_purge, soft_purge, []},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]},
+ {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]}
+ ]
+ },
+ {"4.20",
+ [
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
+ {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
+ {load_module, snmp_conf, soft_purge, soft_purge, []},
+ {load_module, snmp_config, soft_purge, soft_purge, []},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]},
+ {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]}
+ ]
+ },
{"4.19",
[
{load_module, snmpa, soft_purge, soft_purge, []},
- {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
{load_module, snmpa_usm, soft_purge, soft_purge, []},
{load_module, snmpm_usm, soft_purge, soft_purge, []},
{load_module, snmp_log, soft_purge, soft_purge, []},
{load_module, snmp_pdus, soft_purge, soft_purge, []},
{load_module, snmp_conf, soft_purge, soft_purge, []},
- {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpa_conf, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
{load_module, snmp_misc, soft_purge, soft_purge, []},
{load_module, snmp_config, soft_purge, soft_purge, []},
- {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
{load_module, snmpa_trap, soft_purge, soft_purge,
[snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]},
{load_module, snmpa_acm, soft_purge, soft_purge,
[snmp_conf, snmpa_mpd, snmp_target_mib]},
{load_module, snmpa_conf, soft_purge, soft_purge,
- [snmp_notification_mib]},
+ [snmp_config, snmp_notification_mib]},
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
{load_module, snmp_notification_mib, soft_purge, soft_purge,
[snmp_conf, snmp_target_mib]},
{load_module, snmp_community_mib, soft_purge, soft_purge, []},
{load_module, snmp_target_mib, soft_purge, soft_purge,
[snmp_conf]},
- {update, snmpm_net_if, soft, soft_purge, soft_purge, []},
- {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]},
- {update, snmpa_net_if, soft, soft_purge, soft_purge,
- [snmp_conf, snmpa_mpd]},
- {update, snmpa_agent, soft, soft_purge, soft_purge,
- [snmpa_acm, snmpa_mpd, snmpa_trap]}
- ]
- },
- {"4.18",
- [
- {load_module, snmpa, soft_purge, soft_purge, []},
- {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]},
- {load_module, snmpa_usm, soft_purge, soft_purge, []},
- {load_module, snmpm_usm, soft_purge, soft_purge, []},
- {load_module, snmp_misc, soft_purge, soft_purge, []},
- {load_module, snmp_log, soft_purge, soft_purge, []},
- {load_module, snmp_pdus, soft_purge, soft_purge, []},
- {load_module, snmp_conf, soft_purge, soft_purge, []},
- {load_module, snmp_config, soft_purge, soft_purge, [snmp_conf]},
- {load_module, snmpa_conf, soft_purge, soft_purge, []},
- {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]},
- {load_module, snmpa_vacm, soft_purge, soft_purge, []},
- {load_module, snmpa_trap, soft_purge, soft_purge,
- [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]},
- {load_module, snmpa_acm, soft_purge, soft_purge,
- [snmp_conf, snmpa_mpd, snmp_target_mib]},
- {load_module, snmpa, soft_purge, soft_purge,
- [snmp_community_mib,
- snmp_framework_mib,
- snmp_standard_mib,
- snmp_target_mib,
- snmp_user_based_sm_mib,
- snmp_view_based_acm_mib]},
- {load_module, snmp_notification_mib, soft_purge, soft_purge,
- [snmp_conf, snmp_target_mib, snmpa_mib_lib]},
- {load_module, snmp_community_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_framework_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_standard_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_target_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_user_based_sm_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge,
- [snmpa_mib_lib, snmpa_vacm]},
- {load_module, snmpa_mib_lib, soft_purge, soft_purge, []},
-
- {update, snmpm_net_if, soft, soft_purge, soft_purge, []},
- {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]},
-
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
{update, snmpa_net_if, soft, soft_purge, soft_purge,
[snmp_conf, snmpa_mpd]},
{update, snmpa_agent, soft, soft_purge, soft_purge,
@@ -109,84 +109,82 @@
%% ------D o w n g r a d e ---------------------------------------------------
[
+ {"4.20.1",
+ [
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
+ {load_module, snmp_conf, soft_purge, soft_purge, []},
+ {load_module, snmp_config, soft_purge, soft_purge, []},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]},
+ {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]}
+ ]
+ },
+ {"4.20",
+ [
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
+ {load_module, snmp_target_mib, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
+ {load_module, snmp_conf, soft_purge, soft_purge, []},
+ {load_module, snmp_config, soft_purge, soft_purge, []},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_config]},
+ {update, snmpa_agent, soft, soft_purge, soft_purge, [snmpa_mpd]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]}
+ ]
+ },
{"4.19",
[
{load_module, snmpa, soft_purge, soft_purge, []},
- {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]},
+ {load_module, snmpm, soft_purge, soft_purge,
+ [snmpm_server, snmpm_config, snmp_config]},
{load_module, snmpa_usm, soft_purge, soft_purge, []},
{load_module, snmpm_usm, soft_purge, soft_purge, []},
{load_module, snmp_log, soft_purge, soft_purge, []},
{load_module, snmp_pdus, soft_purge, soft_purge, []},
{load_module, snmp_conf, soft_purge, soft_purge, []},
- {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpa_conf, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
{load_module, snmp_misc, soft_purge, soft_purge, []},
{load_module, snmp_config, soft_purge, soft_purge, []},
- {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]},
+ {load_module, snmpa_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config]},
+ {load_module, snmpm_mpd, soft_purge, soft_purge,
+ [snmp_conf, snmp_config, snmpm_config]},
{load_module, snmpa_trap, soft_purge, soft_purge,
[snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]},
{load_module, snmpa_acm, soft_purge, soft_purge,
[snmp_conf, snmpa_mpd, snmp_target_mib]},
{load_module, snmpa_conf, soft_purge, soft_purge,
- [snmp_notification_mib]},
+ [snmp_config, snmp_notification_mib]},
+ {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge, []},
{load_module, snmp_notification_mib, soft_purge, soft_purge,
[snmp_conf, snmp_target_mib]},
{load_module, snmp_community_mib, soft_purge, soft_purge, []},
{load_module, snmp_target_mib, soft_purge, soft_purge,
[snmp_conf]},
-
- {update, snmpm_net_if, soft, soft_purge, soft_purge, []},
- {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]},
-
- {update, snmpa_net_if, soft, soft_purge, soft_purge,
- [snmp_conf, snmpa_mpd]},
- {update, snmpa_agent, soft, soft_purge, soft_purge,
- [snmpa_acm, snmpa_mpd, snmpa_trap]}
- ]
- },
- {"4.18",
- [
- {load_module, snmpa, soft_purge, soft_purge, []},
- {load_module, snmpm, soft_purge, soft_purge, [snmpm_server]},
- {load_module, snmpa_usm, soft_purge, soft_purge, []},
- {load_module, snmpm_usm, soft_purge, soft_purge, []},
- {load_module, snmp_misc, soft_purge, soft_purge, []},
- {load_module, snmp_log, soft_purge, soft_purge, []},
- {load_module, snmp_pdus, soft_purge, soft_purge, []},
- {load_module, snmp_conf, soft_purge, soft_purge, []},
- {load_module, snmpa_conf, soft_purge, soft_purge, [snmp_conf]},
- {load_module, snmp_config, soft_purge, soft_purge, []},
- {load_module, snmpa_mpd, soft_purge, soft_purge, [snmp_conf]},
- {load_module, snmpa_vacm, soft_purge, soft_purge, []},
- {load_module, snmpa_trap, soft_purge, soft_purge,
- [snmpa_mpd, snmp_notification_mib, snmp_target_mib, snmpa_net_if]},
- {load_module, snmpa_acm, soft_purge, soft_purge,
- [snmp_conf, snmpa_mpd, snmp_target_mib]},
- {load_module, snmpa, soft_purge, soft_purge,
- [snmp_community_mib,
- snmp_framework_mib,
- snmp_standard_mib,
- snmp_target_mib,
- snmp_user_based_sm_mib,
- snmp_view_based_acm_mib]},
- {load_module, snmp_notification_mib, soft_purge, soft_purge,
- [snmp_conf, snmp_target_mib, snmpa_mib_lib]},
- {load_module, snmp_community_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_framework_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_standard_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_target_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_user_based_sm_mib, soft_purge, soft_purge,
- [snmpa_mib_lib]},
- {load_module, snmp_view_based_acm_mib, soft_purge, soft_purge,
- [snmpa_mib_lib, snmpa_vacm]},
- {load_module, snmpa_mib_lib, soft_purge, soft_purge, []},
-
- {update, snmpm_net_if, soft, soft_purge, soft_purge, []},
- {update, snmpm_server, soft, soft_purge, soft_purge, [snmpm_net_if]},
-
+ {update, snmpm_net_if, soft, soft_purge, soft_purge,
+ [snmp_conf, snmpm_mpd, snmpm_config]},
+ {update, snmpm_config, soft, soft_purge, soft_purge, [snmp_conf]},
+ {update, snmpm_server, soft, soft_purge, soft_purge,
+ [snmpm_net_if, snmpm_mpd, snmpm_config]},
{update, snmpa_net_if, soft, soft_purge, soft_purge,
[snmp_conf, snmpa_mpd]},
{update, snmpa_agent, soft, soft_purge, soft_purge,
diff --git a/lib/snmp/src/compile/Makefile b/lib/snmp/src/compile/Makefile
index 0ceaf276a6..627af6f185 100644
--- a/lib/snmp/src/compile/Makefile
+++ b/lib/snmp/src/compile/Makefile
@@ -45,11 +45,10 @@ RELSYSDIR = $(RELEASE_PATH)/lib/snmp-$(VSN)
include modules.mk
-ESCRIPT_BIN = $(ESCRIPT_SRC:%.src=$(BIN)/%)
-
-ERL_FILES = $(MODULES:%=%.erl)
-
-TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(ESCRIPT_BIN)
+ESCRIPT_BIN = $(ESCRIPT_SRC:%.src=$(BIN)/%)
+ERL_FILES = $(MODULES:%=%.erl)
+EBIN_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR))
+TARGET_FILES = $(EBIN_FILES) $(ESCRIPT_BIN)
GENERATED_PARSER = $(PARSER_MODULE:%=%.erl)
@@ -125,7 +124,7 @@ release_spec: opt
$(INSTALL_DIR) $(RELSYSDIR)/src/compiler
$(INSTALL_DATA) $(ESCRIPT_SRC) $(PARSER_SRC) $(ERL_FILES) $(INTERNAL_HRL_FILES) $(RELSYSDIR)/src/compiler
$(INSTALL_DIR) $(RELSYSDIR)/ebin
- $(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin
+ $(INSTALL_DATA) $(EBIN_FILES) $(RELSYSDIR)/ebin
$(INSTALL_DIR) $(RELSYSDIR)/bin
$(INSTALL_SCRIPT) $(ESCRIPT_BIN) $(RELSYSDIR)/bin
diff --git a/lib/snmp/src/compile/snmpc.erl b/lib/snmp/src/compile/snmpc.erl
index 195c238184..5e6b81f1ec 100644
--- a/lib/snmp/src/compile/snmpc.erl
+++ b/lib/snmp/src/compile/snmpc.erl
@@ -108,6 +108,7 @@ compile(FileName) ->
%% {i, [import_dir_string()]} ["./"]
%% {il, [import_lib_dir_string()]} []
%% {warnings, bool()} true
+%% warnings_as_errors
%% {outdir, string()} "./"
%% description
%% reference
@@ -199,6 +200,8 @@ get_options([reference|Opts], Formats, Args) ->
get_options(Opts, ["~n reference"|Formats], Args);
get_options([{warnings, Val}|Opts], Formats, Args) ->
get_options(Opts, ["~n warnings: ~w"|Formats], [Val|Args]);
+get_options([warnings_as_errors|Opts], Formats, Args) ->
+ get_options(Opts, ["~n warnings_as_errors"|Formats], Args);
get_options([{verbosity, Val}|Opts], Formats, Args) ->
get_options(Opts, ["~n verbosity: ~w"|Formats], [Val|Args]);
get_options([imports|Opts], Formats, Args) ->
@@ -261,6 +264,8 @@ check_options([{group_check, Atom} | T]) when is_atom(Atom) ->
check_options([{warnings, Bool} | T]) ->
check_bool(warnings, Bool),
check_options(T);
+check_options([warnings_as_errors | T]) ->
+ check_options(T);
check_options([{db, volatile} | T]) ->
check_options(T);
check_options([{db, persistent} | T]) ->
@@ -331,6 +336,9 @@ get_agent_capabilities(Options) ->
get_module_compliance(Options) ->
get_bool_option(module_compliance, Options).
+get_warnings_as_errors(Options) ->
+ lists:member(warnings_as_errors, Options).
+
get_relaxed_row_name_assign_check(Options) ->
lists:member(relaxed_row_name_assign_check, Options).
@@ -409,6 +417,7 @@ init(From, MibFileName, Options) ->
put(reference, get_reference(Options)),
put(agent_capabilities, get_agent_capabilities(Options)),
put(module_compliance, get_module_compliance(Options)),
+ put(warnings_as_errors, get_warnings_as_errors(Options)),
File = filename:rootname(MibFileName, ".mib"),
put(filename, filename:basename(File ++ ".mib")),
R = case catch c_impl(File) of
diff --git a/lib/snmp/src/compile/snmpc.src b/lib/snmp/src/compile/snmpc.src
index 5f9b154bfa..4e91ae9a03 100644
--- a/lib/snmp/src/compile/snmpc.src
+++ b/lib/snmp/src/compile/snmpc.src
@@ -50,7 +50,8 @@
%% The default verbosity (silence) will be filled in
%% during argument processing.
verbosity,
- warnings = false
+ warnings = false,
+ warnings_as_errors = false
}).
@@ -74,6 +75,7 @@
%% --version
%% --verbosity V
%% --warnings
+%% --wae
main(Args) when is_list(Args) ->
case (catch process_args(Args)) of
ok ->
@@ -156,7 +158,8 @@ mk_mib_options(#state{outdir = OutDir,
%% The default verbosity (silence) will be filled in
%% during argument processing.
verbosity = V,
- warnings = W}) ->
+ warnings = W,
+ warnings_as_errors = WAE}) ->
[{outdir, OutDir},
{db, DB},
{i, IDs},
@@ -178,7 +181,8 @@ mk_mib_options(#state{outdir = OutDir,
maybe_option(Imp, imports) ++
maybe_option(MI, module_identity) ++
maybe_option(MC, module_compliance) ++
- maybe_option(AC, agent_capabilities).
+ maybe_option(AC, agent_capabilities) ++
+ maybe_option(WE, warnings_as_errors).
maybe_option(true, Opt) -> [Opt];
maybe_option(_, _) -> [].
@@ -292,6 +296,8 @@ process_args(["--nd"|Args], State) ->
process_args(Args, State#state{no_defaults = true});
process_args(["--rrnac"|Args], State) ->
process_args(Args, State#state{relaxed_row_name_assigne_check = true});
+process_args(["--wae"|Args], State) ->
+ process_args(Args, State#state{warnings_as_errors = true});
process_args([MIB], State) ->
Ext = filename:extension(MIB),
if
@@ -371,6 +377,8 @@ usage() ->
"~n a warning. "
"~n By default it is not included, but if this option is "
"~n present it will be. "
+ "~n --wae - Warnings as errors. "
+ "~n Indicates that warnings shall be treated as errors. "
"~n "
"~n", []),
halt(1).
diff --git a/lib/snmp/src/compile/snmpc_lib.hrl b/lib/snmp/src/compile/snmpc_lib.hrl
index 000486e728..35ec9abd03 100644
--- a/lib/snmp/src/compile/snmpc_lib.hrl
+++ b/lib/snmp/src/compile/snmpc_lib.hrl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2009. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -20,8 +20,17 @@
-ifndef(snmpc_lib).
-define(snmpc_lib, true).
--define(vwarning(F, A), ?verbosity(warning, F, A, ignore)).
--define(vwarning2(F, A, MibLine), ?verbosity(warning, F, A, MibLine)).
+-define(vwarning(F, A),
+ case get(warnings_as_errors) of
+ true -> snmpc_lib:error(F, A);
+ _ -> ?verbosity(warning, F, A, ignore)
+ end).
+
+-define(vwarning2(F, A, MibLine),
+ case get(warnings_as_errors) of
+ true -> snmpc_lib:error(F, A, MibLine);
+ _ -> ?verbosity(warning, F, A, MibLine)
+ end).
-define(vinfo(F, A), ?verbosity(info, F, A, ignore)).
-define(vinfo2(F, A, MibLine), ?verbosity(info, F, A, MibLine)).
-define(vlog(F, A), ?verbosity(log, F, A, ignore)).
diff --git a/lib/snmp/src/manager/snmpm.erl b/lib/snmp/src/manager/snmpm.erl
index 0d084332de..6d2ac8d747 100644
--- a/lib/snmp/src/manager/snmpm.erl
+++ b/lib/snmp/src/manager/snmpm.erl
@@ -50,7 +50,7 @@
register_agent/2, register_agent/3, register_agent/4,
unregister_agent/2, unregister_agent/3,
which_agents/0, which_agents/1,
- agent_info/2, update_agent_info/4,
+ agent_info/2, update_agent_info/3, update_agent_info/4,
register_usm_user/3, unregister_usm_user/2,
which_usm_users/0, which_usm_users/1,
@@ -167,6 +167,7 @@
-include_lib("snmp/include/snmp_types.hrl").
-include("snmpm_atl.hrl").
-include("snmpm_internal.hrl").
+-include("snmp_verbosity.hrl").
-define(DEFAULT_AGENT_PORT, 161).
@@ -447,8 +448,11 @@ agent_info(Addr, Port, Item) ->
Error
end.
+update_agent_info(UserId, TargetName, Info) when is_list(Info) ->
+ snmpm_config:update_agent_info(UserId, TargetName, Info).
+
update_agent_info(UserId, TargetName, Item, Val) ->
- snmpm_config:update_agent_info(UserId, TargetName, Item, Val).
+ update_agent_info(UserId, TargetName, [{Item, Val}]).
%% Backward compatibility functions
update_agent_info(UserId, Addr, Port, Item, Val) ->
diff --git a/lib/snmp/src/manager/snmpm_config.erl b/lib/snmp/src/manager/snmpm_config.erl
index fd6da3e71a..c2e57abddb 100644
--- a/lib/snmp/src/manager/snmpm_config.erl
+++ b/lib/snmp/src/manager/snmpm_config.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2004-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -36,7 +36,8 @@
user_info/0, user_info/1, user_info/2,
register_agent/3, unregister_agent/2,
- agent_info/0, agent_info/2, agent_info/3, update_agent_info/4,
+ agent_info/0, agent_info/2, agent_info/3,
+ update_agent_info/3, update_agent_info/4,
which_agents/0, which_agents/1,
is_known_engine_id/2,
@@ -84,7 +85,9 @@
backup/1,
- mk_target_name/3
+ mk_target_name/3,
+
+ default_transport_domain/0
]).
@@ -127,23 +130,24 @@
%% Macros and Constants:
--define(SERVER, ?MODULE).
--define(BACKUP_DB, snmpm_config_backup).
--define(CONFIG_DB, snmpm_config_db).
+-define(SERVER, ?MODULE).
+-define(BACKUP_DB, snmpm_config_backup).
+-define(CONFIG_DB, snmpm_config_db).
-define(DEFAULT_USER, default_user).
-define(DEFAULT_AGENT_PORT, 161).
--define(IRB_DEFAULT, auto).
-%% -define(IRB_DEFAULT, {user, timer:seconds(15)}).
+-define(IRB_DEFAULT, auto).
+%% -define(IRB_DEFAULT, {user, timer:seconds(15)}).
--define(USER_MOD_DEFAULT, snmpm_user_default).
--define(USER_DATA_DEFAULT, undefined).
+-define(USER_MOD_DEFAULT, snmpm_user_default).
+-define(USER_DATA_DEFAULT, undefined).
%% -define(DEF_ADDR_TAG, default_addr_tag).
-define(DEFAULT_TARGETNAME, default_agent).
--define(DEF_PORT_TAG, default_port_tag).
+-define(DEF_PORT_TAG, default_port_tag).
+-define(SUPPORTED_DOMAINS, [transportDomainUdpIpv4, transportDomainUdpIpv6]).
-ifdef(snmp_debug).
-define(GS_START_LINK(Opts),
@@ -159,6 +163,11 @@
%%%-------------------------------------------------------------------
%%% API
%%%-------------------------------------------------------------------
+
+default_transport_domain() ->
+ transportDomainUdpIpv4.
+
+
start_link(Opts) ->
?d("start_link -> entry with"
"~n Opts: ~p", [Opts]),
@@ -269,9 +278,10 @@ do_user_info(_UserId, BadItem) ->
error({not_found, BadItem}).
-%% A target-name constructed in this way is a string with the following
+%% A target-name constructed in this way is a string with the following:
%% <IP-address>:<Port>-<Version>
-%%
+%% This is intended for backward compatibility and therefor has
+%% only support for IPv4 addresses and *no* other transport domain.
mk_target_name(Addr0, Port, Config) when is_list(Config) ->
Version =
case lists:keysearch(version, 1, Config) of
@@ -280,7 +290,6 @@ mk_target_name(Addr0, Port, Config) when is_list(Config) ->
false ->
select_lowest_supported_version()
end,
-%% p("mk_target_name -> Version: ~p", [Version]),
case normalize_address(Addr0) of
{A, B, C, D} ->
lists:flatten(
@@ -308,57 +317,99 @@ select_lowest_supported_version([H|T], Versions) ->
end.
-register_agent(UserId, _TargetName, _Config) when (UserId =:= user_id) ->
+register_agent(UserId, _TargetName, _Config0) when (UserId =:= user_id) ->
{error, {bad_user_id, UserId}};
-register_agent(UserId, TargetName, Config)
+register_agent(UserId, TargetName, Config0)
when (is_list(TargetName) andalso
(length(TargetName) > 0) andalso
- is_list(Config)) ->
+ is_list(Config0)) ->
-%% p("register_agent -> entry with"
-%% "~n UserId: ~p"
-%% "~n TargetName: ~p"
-%% "~n Config: ~p", [UserId, TargetName, Config]),
+ ?vtrace("register_agent -> entry with"
+ "~n UserId: ~p"
+ "~n TargetName: ~p"
+ "~n Config0: ~p", [UserId, TargetName, Config0]),
%% Check:
%% 1) That the mandatory configs are present
- %% 2) That the illegal config user_id (used internally) is
- %% not present
+ %% 2) That no illegal config, e.g. user_id (used internally),
+ %% is not present
%% 3) Check that there are no invalid or erroneous configs
- %% 4) Chack that the manager is capable to use the selected version
- case verify_agent_config(Config) of
- ok ->
+ %% 4) Check that the manager is capable of using the selected version
+ case verify_agent_config(Config0) of
+ {ok, Config} ->
call({register_agent, UserId, TargetName, Config});
Error ->
Error
end.
-verify_agent_config(Conf) ->
- ok = verify_mandatory(Conf, [engine_id, address, reg_type]),
- case verify_invalid(Conf, [user_id]) of
- ok ->
- case verify_agent_config2(Conf) of
- ok ->
- {ok, Vsns} = system_info(versions),
- Vsn =
- case lists:keysearch(version, 1, Conf) of
- {value, {version, V}} ->
- V;
- false ->
- v1
- end,
- case lists:member(Vsn, Vsns) of
- true ->
- ok;
- false ->
- {error, {version_not_supported_by_manager, Vsn, Vsns}}
- end
- end;
- Error ->
+verify_agent_config(Conf0) ->
+ try
+ begin
+ verify_mandatory(Conf0, [engine_id, address, reg_type]),
+ verify_invalid(Conf0, [user_id]),
+ Conf = verify_agent_config3(Conf0),
+ Vsns = versions(),
+ Vsn = which_version(Conf),
+ verify_version(Vsn, Vsns),
+ {ok, Conf}
+ end
+ catch
+ throw:Error ->
Error
end.
+versions() ->
+ case system_info(versions) of
+ {ok, Vsns} ->
+ Vsns;
+ {error, _} = ERROR ->
+ throw(ERROR)
+ end.
+
+which_version(Conf) ->
+ case lists:keysearch(version, 1, Conf) of
+ {value, {version, V}} ->
+ V;
+ false ->
+ v1
+ end.
+
+verify_version(Vsn, Vsns) ->
+ case lists:member(Vsn, Vsns) of
+ true ->
+ ok;
+ false ->
+ Reason = {version_not_supported_by_manager, Vsn, Vsns},
+ throw({error, Reason})
+ end.
+
+verify_agent_config3(Conf0) ->
+ %% Fix (transport) address and domain
+ {TDomain, Conf1} =
+ case lists:keysearch(tdomain, 1, Conf0) of
+ {value, {tdomain, Dom}} ->
+ {Dom, Conf0};
+ false ->
+ Dom = default_transport_domain(),
+ {Dom, [{tdomain, Dom} | Conf0]}
+ end,
+ Conf2 = case lists:keysearch(address, 1, Conf1) of
+ {value, {address, Address}} ->
+ lists:keyreplace(address, 1, Conf1,
+ {address, {TDomain, Address}});
+ false ->
+ %% This is a mandatory config option,
+ %% a later test will detect this
+ Conf1
+ end,
+ case verify_agent2(Conf2) of
+ {ok, Conf} ->
+ Conf;
+ {error, _} = ERROR ->
+ throw(ERROR)
+ end.
+
verify_agent_config2(Conf) ->
verify_agent2(Conf).
@@ -366,6 +417,7 @@ verify_agent_config2(Conf) ->
unregister_agent(UserId, TargetName) ->
call({unregister_agent, UserId, TargetName}).
+%% This is the old style agent unregistration (using Addr and Port).
unregister_agent(UserId, Addr0, Port) ->
Addr = normalize_address(Addr0),
case do_agent_info(Addr, Port, target_name) of
@@ -421,17 +473,51 @@ which_agents(UserId) ->
Agents = ets:match(snmpm_agent_table, Pat),
[TargetName || [TargetName] <- Agents].
-
-update_agent_info(UserId, TargetName, Item, Val0)
- when (Item =/= user_id) ->
- case (catch verify_val(Item, Val0)) of
- {ok, Val} ->
- call({update_agent_info, UserId, TargetName, Item, Val});
- Error ->
+
+verify_agent_info(TargetName, Info0) ->
+ try
+ begin
+ verify_invalid(Info0, [user_id]),
+ %% Check if address is part of the list and
+ %% if so update it with the domain info.
+ Info =
+ case lists:keysearch(address, 1, Info0) of
+ {value, {address, Addr}} ->
+ %% If domain is part of the info, then use it.
+ %% If not, lookup what is already stored for
+ %% this agent and use that.
+ Domain =
+ case lists:keysearch(tdomain, 1, Info0) of
+ {value, {tdomain, Dom}} ->
+ Dom;
+ false ->
+ {ok, Dom} =
+ agent_info(TargetName, tdomain),
+ Dom
+ end,
+ Addr2 = {Domain, Addr},
+ lists:keyreplace(address, 1, Info0, {address, Addr2});
+ false ->
+ Info0
+ end,
+ verify_agent2(Info)
+ end
+ catch
+ throw:Error ->
Error
end.
-%% Backward compatibillity
+update_agent_info(UserId, TargetName, Info) ->
+ call({update_agent_info, UserId, TargetName, Info}).
+
+%% <BACKWARD-COMPAT-2>
+%% This is wrapped in the interface module, so this function is
+%% only here to catch code-upgrade problems.
+update_agent_info(UserId, TargetName, Item, Val) ->
+ update_agent_info(UserId, TargetName, [{Item, Val}]).
+%% </BACKWARD-COMPAT-2>
+
+%% <BACKWARD-COMPAT-1>
update_agent_info(UserId, Addr, Port, Item, Val) ->
case agent_info(Addr, Port, target_name) of
{ok, TargetName} ->
@@ -439,6 +525,7 @@ update_agent_info(UserId, Addr, Port, Item, Val) ->
Error ->
Error
end.
+%% </BACKWARD-COMPAT-1>
is_known_engine_id(EngineID, TargetName) ->
case agent_info(TargetName, engine_id) of
@@ -650,22 +737,14 @@ unregister_usm_user(EngineID, Name)
call({unregister_usm_user, EngineID, Name}).
verify_usm_user_config(EngineID, Name, Config) ->
- %% case verify_mandatory(Config, []) of
- %% ok ->
- %% case verify_invalid(Config, [engine_id, name]) of
- %% ok ->
- %% verify_usm_user_config2(EngineID, Name, Config);
- %% Error ->
- %% Error
- %% end;
- %% Error ->
- %% Error
- %% end.
- ok = verify_mandatory(Config, []),
- case verify_invalid(Config, [engine_id, name]) of
- ok ->
- verify_usm_user_config2(EngineID, Name, Config);
- Error ->
+ try
+ begin
+ verify_mandatory(Config, []),
+ verify_invalid(Config, [engine_id, name]),
+ verify_usm_user_config2(EngineID, Name, Config)
+ end
+ catch
+ throw:Error ->
Error
end.
@@ -1590,6 +1669,7 @@ check_agent_config2(Agent) ->
throw(Err)
end.
+%% For backward compatibility
check_agent_config({UserId,
TargetName,
Community,
@@ -1597,10 +1677,27 @@ check_agent_config({UserId,
EngineId,
Timeout, MaxMessageSize,
Version, SecModel, SecName, SecLevel}) ->
+ TDomain = default_transport_domain(),
+ check_agent_config({UserId,
+ TargetName,
+ Community,
+ TDomain, Ip, Port,
+ EngineId,
+ Timeout, MaxMessageSize,
+ Version, SecModel, SecName, SecLevel});
+
+check_agent_config({UserId,
+ TargetName,
+ Community,
+ TDomain, Ip, Port,
+ EngineId,
+ Timeout, MaxMessageSize,
+ Version, SecModel, SecName, SecLevel}) ->
?vtrace("check_agent_config -> entry with"
"~n UserId: ~p"
"~n TargetName: ~p"
"~n Community: ~p"
+ "~n TDomain: ~p"
"~n Ip: ~p"
"~n Port: ~p"
"~n EngineId: ~p"
@@ -1610,15 +1707,16 @@ check_agent_config({UserId,
"~n SecModel: ~p"
"~n SecName: ~p"
"~n SecLevel: ~p",
- [UserId, TargetName, Community, Ip, Port,
+ [UserId, TargetName, Community,
+ TDomain, Ip, Port,
EngineId, Timeout, MaxMessageSize,
Version, SecModel, SecName, SecLevel]),
- Addr = normalize_address(Ip),
+ Addr = normalize_address(TDomain, Ip),
?vtrace("check_agent_config -> Addr: ~p", [Addr]),
Agent = {UserId,
TargetName,
Community,
- Addr, Port,
+ TDomain, Addr, Port,
EngineId,
Timeout, MaxMessageSize,
Version, SecModel, SecName, SecLevel},
@@ -1644,6 +1742,7 @@ init_agent_config({UserId, TargetName, Config}) ->
end.
+%% For backward compatibility
verify_agent({UserId,
TargetName,
Comm,
@@ -1651,48 +1750,68 @@ verify_agent({UserId,
EngineId,
Timeout, MMS,
Version, SecModel, SecName, SecLevel}) ->
- ?vtrace("verify_agent -> entry with"
+ TDomain = default_transport_domain(),
+ verify_agent({UserId,
+ TargetName,
+ Comm,
+ TDomain, Ip, Port,
+ EngineId,
+ Timeout, MMS,
+ Version, SecModel, SecName, SecLevel});
+
+verify_agent({UserId,
+ TargetName,
+ Comm,
+ TDomain, Ip, Port,
+ EngineId,
+ Timeout, MMS,
+ Version, SecModel, SecName, SecLevel}) ->
+ ?vdebug("verify_agent -> entry with"
"~n UserId: ~p"
"~n TargetName: ~p", [UserId, TargetName]),
snmp_conf:check_string(TargetName, {gt, 0}),
- case verify_val(address, Ip) of
- {ok, Addr} ->
- snmp_conf:check_integer(Port, {gt, 0}),
- Conf =
- [{reg_type, target_name},
- {address, Addr},
- {port, Port},
- {community, Comm},
- {engine_id, EngineId},
- {timeout, Timeout},
- {max_message_size, MMS},
- {version, Version},
- {sec_model, SecModel},
- {sec_name, SecName},
- {sec_level, SecLevel}
- ],
- case verify_agent2(Conf) of
- ok ->
- {UserId, TargetName, Conf, Version};
- Err ->
- throw(Err)
- end;
-
- Error ->
- ?vlog("verify_agent -> failed: ~n ~p", [Error]),
- throw(Error)
+ snmp_conf:check_integer(Port, {gt, 0}),
+ %% Note that the order of Conf *is* important.
+ %% Some properties may depend on others, so that
+ %% in order to verify one property, another must
+ %% be already verified (and present). An example
+ %% of this is the property 'address', for which
+ %% the property tdomain is needed.
+ Conf0 =
+ [{reg_type, target_name},
+ {tdomain, TDomain},
+ %% This should be taddress, but what the*...
+ {address, {TDomain, Ip}},
+ {port, Port},
+ {community, Comm},
+ {engine_id, EngineId},
+ {timeout, Timeout},
+ {max_message_size, MMS},
+ {version, Version},
+ {sec_model, SecModel},
+ {sec_name, SecName},
+ {sec_level, SecLevel}
+ ],
+ case verify_agent2(Conf0) of
+ {ok, Conf} ->
+ {UserId, TargetName, Conf, Version};
+ Err ->
+ throw(Err)
end.
-verify_agent2([]) ->
- ok;
-verify_agent2([{Item, Val}|Items]) ->
- case verify_val(Item, Val) of
- {ok, _Val} ->
- verify_agent2(Items);
+verify_agent2(Conf) ->
+ verify_agent2(Conf, []).
+
+verify_agent2([], VerifiedConf) ->
+ {ok, VerifiedConf};
+verify_agent2([{Item, Val0}|Items], VerifiedConf) ->
+ case verify_val(Item, Val0) of
+ {ok, Val} ->
+ verify_agent2(Items, [{Item, Val} | VerifiedConf]);
Err ->
Err
end;
-verify_agent2([Bad|_]) ->
+verify_agent2([Bad|_], _VerifiedConf) ->
{error, {bad_agent_config, Bad}}.
@@ -1708,14 +1827,28 @@ read_users_config_file(Dir) ->
check_user_config({Id, Mod, Data}) ->
+ ?vtrace("check_user_config -> entry with"
+ "~n Id: ~p"
+ "~n Mod: ~p"
+ "~n Data: ~p", [Id, Mod, Data]),
check_user_config({Id, Mod, Data, []});
-check_user_config({Id, Mod, _Data, DefaultAgentConfig} = User)
+check_user_config({Id, Mod, Data, DefaultAgentConfig} = _User)
when (Id =/= ?DEFAULT_USER) andalso is_list(DefaultAgentConfig) ->
+ ?vtrace("check_user_config -> entry with"
+ "~n Id: ~p"
+ "~n Mod: ~p"
+ "~n Data: ~p"
+ "~n DefaultAgentConfig: ~p",
+ [Id, Mod, Data, DefaultAgentConfig]),
case (catch verify_user_behaviour(Mod)) of
ok ->
+ ?vtrace("check_user_config -> user behaviour verified", []),
case verify_user_agent_config(DefaultAgentConfig) of
- ok ->
- {ok, User};
+ {ok, DefAgentConf} ->
+ ?vtrace("check_user_config -> "
+ "user agent (default) config verified", []),
+ User2 = {Id, Mod, Data, DefAgentConf},
+ {ok, User2};
{error, Reason} ->
error({bad_default_agent_config, Reason})
end;
@@ -1756,16 +1889,16 @@ verify_user({Id, UserMod, UserData}) ->
verify_user({Id, UserMod, UserData, DefaultAgentConfig})
when (Id =/= ?DEFAULT_USER) andalso is_list(DefaultAgentConfig) ->
?d("verify_user -> entry with"
- "~n Id: ~p"
- "~n UserMod: ~p"
- "~n UserData: ~p"
+ "~n Id: ~p"
+ "~n UserMod: ~p"
+ "~n UserData: ~p"
"~n DefaultAgentConfig: ~p",
[Id, UserMod, UserData, DefaultAgentConfig]),
case (catch verify_user_behaviour(UserMod)) of
ok ->
case verify_user_agent_config(DefaultAgentConfig) of
- ok ->
- Config = default_agent_config(DefaultAgentConfig),
+ {ok, DefAgentConf} ->
+ Config = default_agent_config(DefAgentConf),
{ok, #user{id = Id,
mod = UserMod,
data = UserData,
@@ -1783,10 +1916,15 @@ verify_user({Id, _, _, _}) ->
{error, {bad_user_id, Id}}.
verify_user_agent_config(Conf) ->
- case verify_invalid(Conf, [user_id, engine_id, address]) of
- ok ->
- verify_agent_config2(Conf);
- Error ->
+ try
+ begin
+ verify_invalid(Conf, [user_id, engine_id, address]),
+ verify_agent_config2(Conf)
+ end
+ catch
+ throw:Error ->
+ ?vdebug("verify_user_agent_config -> throw"
+ "~n Error: ~p", [Error]),
Error
end.
@@ -2147,6 +2285,16 @@ handle_call({unregister_agent, UserId, TargetName}, _From, State) ->
Reply = handle_unregister_agent(UserId, TargetName),
{reply, Reply, State};
+handle_call({update_agent_info, UserId, TargetName, Info},
+ _From, State) ->
+ ?vlog("received update_agent_info request: "
+ "~n UserId: ~p"
+ "~n TargetName: ~p"
+ "~n Info: ~p", [UserId, TargetName, Info]),
+ Reply = handle_update_agent_info(UserId, TargetName, Info),
+ {reply, Reply, State};
+
+%% <BACKWARD-COMPAT>
handle_call({update_agent_info, UserId, TargetName, Item, Val},
_From, State) ->
?vlog("received update_agent_info request: "
@@ -2156,6 +2304,7 @@ handle_call({update_agent_info, UserId, TargetName, Item, Val},
"~n Val: ~p", [UserId, TargetName, Item, Val]),
Reply = handle_update_agent_info(UserId, TargetName, Item, Val),
{reply, Reply, State};
+%% </BACKWARD-COMPAT>
handle_call({register_usm_user, User}, _From, State) ->
?vlog("received register_usm_user request: "
@@ -2517,16 +2666,27 @@ handle_register_agent(UserId, TargetName, Config) ->
"~n Config: ~p", [UserId, TargetName, Config]),
case (catch agent_info(TargetName, user_id)) of
{error, _} ->
+ ?vtrace("handle_register_agent -> user_id not found in config", []),
case ets:lookup(snmpm_user_table, UserId) of
[#user{default_agent_config = DefConfig}] ->
+ ?vtrace("handle_register_agent -> "
+ "~n DefConfig: ~p", [DefConfig]),
+ %% First, insert this users default config
+ ?vtrace("handle_register_agent -> store default config", []),
do_handle_register_agent(TargetName, DefConfig),
+ %% Second, insert the config for this agent
+ ?vtrace("handle_register_agent -> store config", []),
do_handle_register_agent(TargetName,
[{user_id, UserId}|Config]),
%% <DIRTY-BACKWARD-COMPATIBILLITY>
%% And now for some (backward compatibillity)
%% dirty crossref stuff
+ ?vtrace("handle_register_agent -> lookup address", []),
{ok, Addr} = agent_info(TargetName, address),
+ ?vtrace("handle_register_agent -> Addr: ~p, lookup Port",
+ [Addr]),
{ok, Port} = agent_info(TargetName, port),
+ ?vtrace("handle_register_agent -> register cross-ref fix", []),
ets:insert(snmpm_agent_table,
{{Addr, Port, target_name}, TargetName}),
%% </DIRTY-BACKWARD-COMPATIBILLITY>
@@ -2551,10 +2711,18 @@ handle_register_agent(UserId, TargetName, Config) ->
do_handle_register_agent(_TargetName, []) ->
ok;
do_handle_register_agent(TargetName, [{Item, Val}|Rest]) ->
+ ?vtrace("handle_register_agent -> entry with"
+ "~n TargetName: ~p"
+ "~n Item: ~p"
+ "~n Val: ~p"
+ "~n Rest: ~p", [TargetName, Item, Val, Rest]),
case (catch do_update_agent_info(TargetName, Item, Val)) of
ok ->
do_handle_register_agent(TargetName, Rest);
{error, Reason} ->
+ ?vtrace("handle_register_agent -> failed updating ~p"
+ "~n Item: ~p"
+ "~n Reason: ~p", [Item, Reason]),
ets:match_delete(snmpm_agent_table, {TargetName, '_'}),
{error, Reason}
end;
@@ -2589,41 +2757,61 @@ handle_unregister_agent(UserId, TargetName) ->
end.
-handle_update_agent_info(UserId, TargetName, Item, Val) ->
+handle_update_agent_info(UserId, TargetName, Info) ->
?vdebug("handle_update_agent_info -> entry with"
"~n UserId: ~p"
"~n TargetName: ~p"
- "~n Item: ~p"
- "~n Val: ~p", [UserId, TargetName, Item, Val]),
+ "~n Info: ~p", [UserId, TargetName, Info]),
+ %% Verify ownership
case (catch agent_info(TargetName, user_id)) of
- {ok, UserId} ->
- do_update_agent_info(TargetName, Item, Val);
+ {ok, UserId} ->
+ handle_update_agent_info(TargetName, Info);
{ok, OtherUserId} ->
{error, {not_owner, OtherUserId}};
Error ->
Error
end.
-do_update_agent_info(TargetName, Item, Val0) ->
-%% p("do_update_agent_info -> entry with"
-%% "~n TargetName: ~p"
-%% "~n Item: ~p"
-%% "~n Val0: ~p", [TargetName, Item, Val0]),
- case verify_val(Item, Val0) of
- {ok, Val} ->
-%% p("do_update_agent_info -> verified value"
-%% "~n Val: ~p", [Val]),
- ets:insert(snmpm_agent_table, {{TargetName, Item}, Val}),
- ok;
+handle_update_agent_info(TargetName, Info0) ->
+ ?vtrace("handle_update_agent_info -> entry with"
+ "~n TargetName: ~p"
+ "~n Info0: ~p", [TargetName, Info0]),
+ %% Verify info
+ try verify_agent_info(TargetName, Info0) of
+ {ok, Info} ->
+ do_update_agent_info(TargetName, Info);
Error ->
- ?vlog("do_update_agent_info -> verify value failed: "
- "~n TargetName: ~p"
- "~n Item: ~p"
- "~n Val0: ~p"
- "~n Error: ~p", [TargetName, Item, Val0, Error]),
- {error, {bad_agent_val, TargetName, Item, Val0}}
+ Error
+ catch
+ throw:Error ->
+ Error;
+ T:E ->
+ {error, {failed_info_verification, Info0, T, E}}
end.
+handle_update_agent_info(UserId, TargetName, Item, Val) ->
+ ?vdebug("handle_update_agent_info -> entry with"
+ "~n UserId: ~p"
+ "~n TargetName: ~p"
+ "~n Item: ~p"
+ "~n Val: ~p", [UserId, TargetName, Item, Val]),
+ handle_update_agent_info(TargetName, [{Item, Val}]).
+
+do_update_agent_info(TargetName, Info) ->
+ InsertItem =
+ fun({Item, Val}) ->
+ ets:insert(snmpm_agent_table, {{TargetName, Item}, Val})
+ end,
+ lists:foreach(InsertItem, Info).
+
+do_update_agent_info(TargetName, Item, Val) ->
+ ?vtrace("do_update_agent_info -> entry with"
+ "~n TargetName: ~p"
+ "~n Item: ~p"
+ "~n Val: ~p", [TargetName, Item, Val]),
+ ets:insert(snmpm_agent_table, {{TargetName, Item}, Val}),
+ ok.
+
handle_register_usm_user(#usm_user{engine_id = EngineID,
name = Name} = User) ->
@@ -2791,7 +2979,7 @@ verify_mandatory(Conf, [Mand|Mands]) ->
true ->
verify_mandatory(Conf, Mands);
false ->
- {error, {missing_mandatory_config, Mand}}
+ throw({error, {missing_mandatory_config, Mand}})
end.
verify_invalid(_, []) ->
@@ -2801,7 +2989,7 @@ verify_invalid(Conf, [Inv|Invs]) ->
false ->
verify_invalid(Conf, Invs);
true ->
- {error, {illegal_config, Inv}}
+ throw({error, {illegal_config, Inv}})
end.
@@ -2810,10 +2998,26 @@ verify_val(user_id, UserId) ->
verify_val(reg_type, RegType)
when (RegType =:= addr_port) orelse (RegType =:= target_name) ->
{ok, RegType};
-verify_val(address, Addr0) ->
- case normalize_address(Addr0) of
+verify_val(tdomain = Item, snmpUDPDomain = _Domain) ->
+ verify_val(Item, transportDomainUdpIpv4);
+verify_val(tdomain, Domain) ->
+ case lists:member(Domain, ?SUPPORTED_DOMAINS) of
+ true ->
+ {ok, Domain};
+ false ->
+ case lists:member(Domain, snmp_conf:all_domains()) of
+ true ->
+ error({unsupported_domain, Domain});
+ false ->
+ error({unknown_domain, Domain})
+ end
+ end;
+verify_val(address, {Domain, Addr0}) ->
+ case normalize_address(Domain, Addr0) of
{_A1, _A2, _A3, _A4} = Addr ->
{ok, Addr};
+ {_A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8} = Addr ->
+ {ok, Addr};
_ when is_list(Addr0) ->
case (catch snmp_conf:check_ip(Addr0)) of
ok ->
@@ -2824,6 +3028,8 @@ verify_val(address, Addr0) ->
_ ->
error({bad_address, Addr0})
end;
+verify_val(address, BadAddress) ->
+ error({bad_address, BadAddress});
verify_val(port, Port) ->
case (catch snmp_conf:check_integer(Port, {gt, 0})) of
ok ->
@@ -2875,7 +3081,7 @@ verify_val(sec_name, BadName) ->
verify_val(sec_level, Level) ->
(catch snmp_conf:check_sec_level(Level));
verify_val(Item, _) ->
- {error, {no_such_item, Item}}.
+ {error, {unknown_item, Item}}.
%%%-------------------------------------------------------------------
@@ -3034,11 +3240,17 @@ init_mini_mib_elems(MibName, [_|T], Res) ->
%%----------------------------------------------------------------------
normalize_address(Addr) ->
- case inet:getaddr(Addr, inet) of
+ normalize_address(snmpUDPDomain, Addr).
+
+normalize_address(snmpUDPDomain, Addr) ->
+ normalize_address(transportDomainUdpIpv4, Addr);
+
+normalize_address(Domain, Addr) ->
+ case inet:getaddr(Addr, td2fam(Domain)) of
{ok, Addr2} ->
Addr2;
_ when is_list(Addr) ->
- case (catch snmp_conf:check_ip(Addr)) of
+ case (catch snmp_conf:check_ip(Domain, Addr)) of
ok ->
list_to_tuple(Addr);
_ ->
@@ -3048,6 +3260,9 @@ normalize_address(Addr) ->
Addr
end.
+td2fam(transportDomainUdpIpv4) -> inet;
+td2fam(transportDomainUdpIpv6) -> inet6.
+
%%----------------------------------------------------------------------
diff --git a/lib/snmp/src/manager/snmpm_mpd.erl b/lib/snmp/src/manager/snmpm_mpd.erl
index 7712370d28..627838e3d4 100644
--- a/lib/snmp/src/manager/snmpm_mpd.erl
+++ b/lib/snmp/src/manager/snmpm_mpd.erl
@@ -92,7 +92,7 @@ reset(#state{v3 = V3}) ->
%% Purpose: This is the main Message Dispatching function. (see
%% section 4.2.1 in rfc2272)
%%-----------------------------------------------------------------
-process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) ->
+process_msg(Msg, Domain, Addr, Port, State, NoteStore, Logger) ->
inc(snmpInPkts),
@@ -102,18 +102,18 @@ process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) ->
#message{version = 'version-1', vsn_hdr = Community, data = Data}
when State#state.v1 =:= true ->
HS = ?empty_msg_size + length(Community),
- process_v1_v2c_msg('version-1', NoteStore, Msg, TDomain,
- Addr, Port,
+ process_v1_v2c_msg('version-1', NoteStore, Msg,
+ Domain, Addr, Port,
Community, Data, HS, Logger);
%% Version 2
#message{version = 'version-2', vsn_hdr = Community, data = Data}
when State#state.v2c =:= true ->
HS = ?empty_msg_size + length(Community),
- process_v1_v2c_msg('version-2', NoteStore, Msg, TDomain,
- Addr, Port,
- Community, Data, HS, Logger);
-
+ (catch process_v1_v2c_msg('version-2', NoteStore, Msg,
+ Domain, Addr, Port,
+ Community, Data, HS, Logger));
+
%% Version 3
#message{version = 'version-3', vsn_hdr = H, data = Data}
when State#state.v3 =:= true ->
@@ -148,17 +148,30 @@ process_msg(Msg, TDomain, Addr, Port, State, NoteStore, Logger) ->
%%-----------------------------------------------------------------
%% Handles a Community based message (v1 or v2c).
%%-----------------------------------------------------------------
-process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain,
+process_v1_v2c_msg(Vsn, _NoteStore, Msg, Domain,
Addr, Port,
Community, Data, HS, Log) ->
?vdebug("process_v1_v2c_msg -> entry with"
"~n Vsn: ~p"
+ "~n Domain: ~p"
"~n Addr: ~p"
"~n Port: ~p"
"~n Community: ~p"
- "~n HS: ~p", [Vsn, Addr, Port, Community, HS]),
+ "~n HS: ~p", [Vsn, Domain, Addr, Port, Community, HS]),
+ {TDomain, TAddress} =
+ try
+ begin
+ TD = snmp_conf:mk_tdomain(Domain),
+ TA = snmp_conf:mk_taddress(Domain, Addr, Port),
+ {TD, TA}
+ end
+ catch
+ throw:{error, TReason} ->
+ throw({discarded, {badarg, Domain, TReason}})
+ end,
+
Max = get_max_message_size(),
AgentMax = get_agent_max_message_size(Addr, Port),
PduMS = pdu_ms(Max, AgentMax, HS),
@@ -170,14 +183,14 @@ process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain,
?vtrace("process_v1_v2c_msg -> was a pdu", []),
Log(Msg),
inc_snmp_in(Pdu),
- MsgData = {Community, sec_model(Vsn)},
+ MsgData = {Community, sec_model(Vsn), TDomain, TAddress},
{ok, Vsn, Pdu, PduMS, MsgData};
Trap when is_record(Trap, trappdu) ->
?vtrace("process_v1_v2c_msg -> was a trap", []),
Log(Msg),
inc_snmp_in(Trap),
- MsgData = {Community, sec_model(Vsn)},
+ MsgData = {Community, sec_model(Vsn), TDomain, TAddress},
{ok, Vsn, Trap, PduMS, MsgData};
{'EXIT', Reason} ->
@@ -185,11 +198,7 @@ process_v1_v2c_msg(Vsn, _NoteStore, Msg, snmpUDPDomain,
"~n Reason: ~p", [Reason]),
inc(snmpInASNParseErrs),
{discarded, Reason}
- end;
-process_v1_v2c_msg(_Vsn, _NoteStore, _Msg, TDomain,
- _Addr, _Port,
- _Comm, _HS, _Data, _Log) ->
- {discarded, {badarg, TDomain}}.
+ end.
pdu_ms(MgrMMS, AgentMMS, HS) when AgentMMS < MgrMMS ->
AgentMMS - HS;
@@ -482,8 +491,8 @@ generate_msg('version-3', NoteStore, Pdu,
generate_v3_msg(NoteStore, Pdu,
SecModel, SecName, SecLevel, CtxEngineID, CtxName,
TargetName, Log);
-generate_msg(Vsn, _NoteStore, Pdu, {Community, _SecModel}, Log) ->
- generate_v1_v2c_msg(Vsn, Pdu, Community, Log).
+generate_msg(Vsn, _NoteStore, Pdu, {Comm, _SecModel}, Log) ->
+ generate_v1_v2c_msg(Vsn, Pdu, Comm, Log).
generate_v3_msg(NoteStore, Pdu,
@@ -627,6 +636,8 @@ generate_response_msg('version-3', Pdu,
generate_v3_response_msg(Pdu, MsgID, SecModel, SecName, SecLevel,
CtxEngineID, CtxName, SecData, Log);
generate_response_msg(Vsn, Pdu, {Comm, _SecModel}, Log) ->
+ generate_v1_v2c_response_msg(Vsn, Pdu, Comm, Log);
+generate_response_msg(Vsn, Pdu, {Comm, _SecModel, _TDomain, _TAddress}, Log) ->
generate_v1_v2c_response_msg(Vsn, Pdu, Comm, Log).
diff --git a/lib/snmp/src/manager/snmpm_net_if.erl b/lib/snmp/src/manager/snmpm_net_if.erl
index a116c9f26b..4d6bd9aa33 100644
--- a/lib/snmp/src/manager/snmpm_net_if.erl
+++ b/lib/snmp/src/manager/snmpm_net_if.erl
@@ -28,7 +28,8 @@
start_link/2,
stop/1,
send_pdu/6, % Backward compatibillity
- send_pdu/7,
+ send_pdu/7, % Backward compatibillity
+ send_pdu/8,
inform_response/4,
@@ -101,16 +102,21 @@ stop(Pid) ->
send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port) ->
send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ?DEFAULT_EXTRA_INFO).
-send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo)
+send_pdu(Pid, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo) ->
+ Domain = snmpm_config:default_transport_domain(),
+ send_pdu(Pid, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo).
+
+send_pdu(Pid, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo)
when is_record(Pdu, pdu) ->
?d("send_pdu -> entry with"
"~n Pid: ~p"
"~n Pdu: ~p"
"~n Vsn: ~p"
"~n MsgData: ~p"
+ "~n Domain: ~p"
"~n Addr: ~p"
- "~n Port: ~p", [Pid, Pdu, Vsn, MsgData, Addr, Port]),
- cast(Pid, {send_pdu, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo}).
+ "~n Port: ~p", [Pid, Pdu, Vsn, MsgData, Domain, Addr, Port]),
+ cast(Pid, {send_pdu, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo}).
note_store(Pid, NoteStore) ->
call(Pid, {note_store, NoteStore}).
@@ -380,15 +386,17 @@ handle_call(Req, From, State) ->
%% {noreply, State, Timeout} |
%% {stop, Reason, State} (terminate/2 is called)
%%--------------------------------------------------------------------
-handle_cast({send_pdu, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo}, State) ->
+handle_cast({send_pdu, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo},
+ State) ->
?vlog("received send_pdu message with"
"~n Pdu: ~p"
"~n Vsn: ~p"
"~n MsgData: ~p"
+ "~n Domain: ~p"
"~n Addr: ~p"
- "~n Port: ~p", [Pdu, Vsn, MsgData, Addr, Port]),
+ "~n Port: ~p", [Pdu, Vsn, MsgData, Domain, Addr, Port]),
maybe_process_extra_info(ExtraInfo),
- maybe_handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, State),
+ maybe_handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port, State),
{noreply, State};
handle_cast({inform_response, Ref, Addr, Port}, State) ->
@@ -545,8 +553,9 @@ handle_recv_msg(Addr, Port, Bytes,
mpd_state = MpdState,
sock = Sock,
log = Log} = State) ->
+ Domain = snmp_conf:which_domain(Addr), % What the ****...
Logger = logger(Log, read, Addr, Port),
- case (catch snmpm_mpd:process_msg(Bytes, snmpUDPDomain, Addr, Port,
+ case (catch snmpm_mpd:process_msg(Bytes, Domain, Addr, Port,
MpdState, NoteStore, Logger)) of
{ok, Vsn, Pdu, MS, ACM} ->
@@ -734,17 +743,17 @@ irgc_stop(Ref) ->
(catch erlang:cancel_timer(Ref)).
-maybe_handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port,
+maybe_handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port,
#state{filter = FilterMod} = State) ->
case (catch FilterMod:accept_send_pdu(Addr, Port, pdu_type_of(Pdu))) of
false ->
inc(netIfPduOutDrops),
ok;
_ ->
- handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port, State)
+ handle_send_pdu(Pdu, Vsn, MsgData, Domain, Addr, Port, State)
end.
-handle_send_pdu(Pdu, Vsn, MsgData, Addr, Port,
+handle_send_pdu(Pdu, Vsn, MsgData, _Domain, Addr, Port,
#state{server = Pid,
note_store = NoteStore,
sock = Sock,
diff --git a/lib/snmp/src/manager/snmpm_server.erl b/lib/snmp/src/manager/snmpm_server.erl
index 58a58507d6..484954addb 100644
--- a/lib/snmp/src/manager/snmpm_server.erl
+++ b/lib/snmp/src/manager/snmpm_server.erl
@@ -161,7 +161,8 @@
{id,
user_id,
reg_type,
- target,
+ target,
+ domain,
addr,
port,
type,
@@ -1175,11 +1176,12 @@ handle_sync_get(Pid, UserId, TargetName, Oids, SendOpts, From, State) ->
"~n From: ~p",
[Pid, UserId, TargetName, Oids, SendOpts, From]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_sync_get -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
ReqId = send_get_request(Oids, Vsn, MsgData,
- Addr, Port, Extra, State),
+ Domain, Addr, Port,
+ Extra, State),
?vdebug("handle_sync_get -> ReqId: ~p", [ReqId]),
Msg = {sync_timeout, ReqId, From},
Timeout = ?SYNC_GET_TIMEOUT(SendOpts),
@@ -1190,6 +1192,7 @@ handle_sync_get(Pid, UserId, TargetName, Oids, SendOpts, From, State) ->
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get,
@@ -1227,11 +1230,12 @@ handle_sync_get_next(Pid, UserId, TargetName, Oids, SendOpts,
"~n From: ~p",
[Pid, UserId, TargetName, Oids, SendOpts, From]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_sync_get_next -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
ReqId = send_get_next_request(Oids, Vsn, MsgData,
- Addr, Port, Extra, State),
+ Domain, Addr, Port,
+ Extra, State),
?vdebug("handle_sync_get_next -> ReqId: ~p", [ReqId]),
Msg = {sync_timeout, ReqId, From},
Timeout = ?SYNC_GET_NEXT_TIMEOUT(SendOpts),
@@ -1242,6 +1246,7 @@ handle_sync_get_next(Pid, UserId, TargetName, Oids, SendOpts,
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get_next,
@@ -1285,10 +1290,11 @@ handle_sync_get_bulk(Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts,
"~n From: ~p",
[Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts, From]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_sync_get_bulk -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
- ReqId = send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port,
+ ReqId = send_get_bulk_request(Oids, Vsn, MsgData,
+ Domain, Addr, Port,
NonRep, MaxRep, Extra, State),
?vdebug("handle_sync_get_bulk -> ReqId: ~p", [ReqId]),
Msg = {sync_timeout, ReqId, From},
@@ -1300,6 +1306,7 @@ handle_sync_get_bulk(Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts,
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get_bulk,
@@ -1339,11 +1346,12 @@ handle_sync_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, From, State) ->
"~n From: ~p",
[Pid, UserId, TargetName, VarsAndVals, From]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_sync_set -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
ReqId = send_set_request(VarsAndVals, Vsn, MsgData,
- Addr, Port, Extra, State),
+ Domain, Addr, Port,
+ Extra, State),
?vdebug("handle_sync_set -> ReqId: ~p", [ReqId]),
Msg = {sync_timeout, ReqId, From},
Timeout = ?SYNC_SET_TIMEOUT(SendOpts),
@@ -1354,6 +1362,7 @@ handle_sync_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, From, State) ->
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = set,
@@ -1391,10 +1400,11 @@ handle_async_get(Pid, UserId, TargetName, Oids, SendOpts, State) ->
"~n SendOpts: ~p",
[Pid, UserId, TargetName, Oids, SendOpts]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_async_get -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
- ReqId = send_get_request(Oids, Vsn, MsgData, Addr, Port,
+ ReqId = send_get_request(Oids, Vsn, MsgData,
+ Domain, Addr, Port,
Extra, State),
?vdebug("handle_async_get -> ReqId: ~p", [ReqId]),
Expire = ?ASYNC_GET_TIMEOUT(SendOpts),
@@ -1402,6 +1412,7 @@ handle_async_get(Pid, UserId, TargetName, Oids, SendOpts, State) ->
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get,
@@ -1439,17 +1450,19 @@ handle_async_get_next(Pid, UserId, TargetName, Oids, SendOpts, State) ->
"~n SendOpts: ~p",
[Pid, UserId, TargetName, Oids, SendOpts]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_async_get_next -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
ReqId = send_get_next_request(Oids, Vsn, MsgData,
- Addr, Port, Extra, State),
+ Domain, Addr, Port,
+ Extra, State),
?vdebug("handle_async_get_next -> ReqId: ~p", [ReqId]),
Expire = ?ASYNC_GET_NEXT_TIMEOUT(SendOpts),
Req = #request{id = ReqId,
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get_next,
@@ -1494,10 +1507,11 @@ handle_async_get_bulk(Pid,
"~n SendOpts: ~p",
[Pid, UserId, TargetName, NonRep, MaxRep, Oids, SendOpts]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_async_get_bulk -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
- ReqId = send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port,
+ ReqId = send_get_bulk_request(Oids, Vsn, MsgData,
+ Domain, Addr, Port,
NonRep, MaxRep, Extra, State),
?vdebug("handle_async_get_bulk -> ReqId: ~p", [ReqId]),
Expire = ?ASYNC_GET_BULK_TIMEOUT(SendOpts),
@@ -1505,6 +1519,7 @@ handle_async_get_bulk(Pid,
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = get_bulk,
@@ -1541,17 +1556,19 @@ handle_async_set(Pid, UserId, TargetName, VarsAndVals, SendOpts, State) ->
"~n SendOpts: ~p",
[Pid, UserId, TargetName, VarsAndVals, SendOpts]),
case agent_data(TargetName, SendOpts) of
- {ok, RegType, Addr, Port, Vsn, MsgData} ->
+ {ok, RegType, Domain, Addr, Port, Vsn, MsgData} ->
?vtrace("handle_async_set -> send a ~p message", [Vsn]),
Extra = ?GET_EXTRA(SendOpts),
ReqId = send_set_request(VarsAndVals, Vsn, MsgData,
- Addr, Port, Extra, State),
+ Domain, Addr, Port,
+ Extra, State),
?vdebug("handle_async_set -> ReqId: ~p", [ReqId]),
Expire = ?ASYNC_SET_TIMEOUT(SendOpts),
Req = #request{id = ReqId,
user_id = UserId,
reg_type = RegType,
target = TargetName,
+ domain = Domain,
addr = Addr,
port = Port,
type = set,
@@ -2907,7 +2924,7 @@ do_gc(Key, Now) ->
%%
%%----------------------------------------------------------------------
-send_get_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo,
+send_get_request(Oids, Vsn, MsgData, Domain, Addr, Port, ExtraInfo,
#state{net_if = NetIf,
net_if_mod = Mod,
mini_mib = MiniMIB}) ->
@@ -2918,34 +2935,39 @@ send_get_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo,
"~n Pdu: ~p"
"~n Vsn: ~p"
"~n MsgData: ~p"
+ "~n Domain: ~p"
"~n Addr: ~p"
- "~n Port: ~p", [Mod, NetIf, Pdu, Vsn, MsgData, Addr, Port]),
- (catch Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo)),
+ "~n Port: ~p",
+ [Mod, NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port]),
+ Res = (catch Mod:send_pdu(NetIf, Pdu, Vsn, MsgData,
+ Domain, Addr, Port, ExtraInfo)),
+ ?vtrace("send_get_request -> send result:"
+ "~n ~p", [Res]),
Pdu#pdu.request_id.
-send_get_next_request(Oids, Vsn, MsgData, Addr, Port, ExtraInfo,
+send_get_next_request(Oids, Vsn, MsgData, Domain, Addr, Port, ExtraInfo,
#state{mini_mib = MiniMIB,
net_if = NetIf,
net_if_mod = Mod}) ->
Pdu = make_pdu(get_next, Oids, MiniMIB),
- Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo),
+ Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo),
Pdu#pdu.request_id.
-send_get_bulk_request(Oids, Vsn, MsgData, Addr, Port,
+send_get_bulk_request(Oids, Vsn, MsgData, Domain, Addr, Port,
NonRep, MaxRep, ExtraInfo,
#state{mini_mib = MiniMIB,
net_if = NetIf,
net_if_mod = Mod}) ->
Pdu = make_pdu(bulk, {NonRep, MaxRep, Oids}, MiniMIB),
- Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo),
+ Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo),
Pdu#pdu.request_id.
-send_set_request(VarsAndVals, Vsn, MsgData, Addr, Port, ExtraInfo,
+send_set_request(VarsAndVals, Vsn, MsgData, Domain, Addr, Port, ExtraInfo,
#state{mini_mib = MiniMIB,
net_if = NetIf,
net_if_mod = Mod}) ->
Pdu = make_pdu(set, VarsAndVals, MiniMIB),
- Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Addr, Port, ExtraInfo),
+ Mod:send_pdu(NetIf, Pdu, Vsn, MsgData, Domain, Addr, Port, ExtraInfo),
Pdu#pdu.request_id.
%% send_discovery(Vsn, MsgData, Addr, Port, ExtraInfo,
@@ -3181,10 +3203,11 @@ agent_data(TargetName, SendOpts) ->
{Comm, SecModel}
end,
+ Domain = agent_data_item(tdomain, Info),
Addr = agent_data_item(address, Info),
Port = agent_data_item(port, Info),
RegType = agent_data_item(reg_type, Info),
- {ok, RegType, Addr, Port, version(Version), MsgData};
+ {ok, RegType, Domain, Addr, Port, version(Version), MsgData};
Error ->
Error
end.
diff --git a/lib/snmp/src/misc/snmp_conf.erl b/lib/snmp/src/misc/snmp_conf.erl
index 20f4455d10..7249def24e 100644
--- a/lib/snmp/src/misc/snmp_conf.erl
+++ b/lib/snmp/src/misc/snmp_conf.erl
@@ -37,7 +37,9 @@
check_timer/1,
+ all_domains/0,
check_domain/1,
+ all_tdomains/0,
check_tdomain/1,
mk_tdomain/1,
which_domain/1,
@@ -345,6 +347,25 @@ check_sec_level(BadSecLevel) ->
%% ---------
+all_tdomains() ->
+ [
+ ?transportDomainUdpIpv4,
+ ?transportDomainUdpIpv6,
+ ?transportDomainUdpIpv4z,
+ ?transportDomainUdpIpv6z,
+ ?transportDomainTcpIpv4,
+ ?transportDomainTcpIpv6,
+ ?transportDomainTcpIpv4z,
+ ?transportDomainTcpIpv6z,
+ ?transportDomainSctpIpv4,
+ ?transportDomainSctpIpv6,
+ ?transportDomainSctpIpv4z,
+ ?transportDomainSctpIpv6z,
+ ?transportDomainLocal,
+ ?transportDomainUdpDns,
+ ?transportDomainTcpDns,
+ ?transportDomainSctpDns
+ ].
check_tdomain(TDomain) ->
SupportedTDomains =
@@ -353,25 +374,7 @@ check_tdomain(TDomain) ->
?transportDomainUdpIpv4,
?transportDomainUdpIpv6
],
- AllTDomains =
- [
- ?transportDomainUdpIpv4,
- ?transportDomainUdpIpv6,
- ?transportDomainUdpIpv4z,
- ?transportDomainUdpIpv6z,
- ?transportDomainTcpIpv4,
- ?transportDomainTcpIpv6,
- ?transportDomainTcpIpv4z,
- ?transportDomainTcpIpv6z,
- ?transportDomainSctpIpv4,
- ?transportDomainSctpIpv6,
- ?transportDomainSctpIpv4z,
- ?transportDomainSctpIpv6z,
- ?transportDomainLocal,
- ?transportDomainUdpDns,
- ?transportDomainTcpDns,
- ?transportDomainSctpDns
- ],
+ AllTDomains = all_tdomains(),
case lists:member(TDomain, SupportedTDomains) of
true ->
ok;
@@ -388,7 +391,7 @@ check_tdomain(TDomain) ->
%% ---------
mk_tdomain(snmpUDPDomain) ->
- ?snmpUDPDomain;
+ mk_tdomain(transportDomainUdpIpv4);
mk_tdomain(transportDomainUdpIpv4) ->
?transportDomainUdpIpv4;
mk_tdomain(transportDomainUdpIpv6) ->
@@ -474,6 +477,26 @@ do_check_timer(WaitFor, Factor, Incr, Retry) ->
%% ---------
+all_domains() ->
+ [
+ transportDomainUdpIpv4,
+ transportDomainUdpIpv6,
+ transportDomainUdpIpv4z,
+ transportDomainUdpIpv6z,
+ transportDomainTcpIpv4,
+ transportDomainTcpIpv6,
+ transportDomainTcpIpv4z,
+ transportDomainTcpIpv6z,
+ transportDomainSctpIpv4,
+ transportDomainSctpIpv6,
+ transportDomainSctpIpv4z,
+ transportDomainSctpIpv6z,
+ transportDomainLocal,
+ transportDomainUdpDns,
+ transportDomainTcpDns,
+ transportDomainSctpDns
+ ].
+
check_domain(Domain) ->
SupportedDomains =
[
@@ -481,25 +504,7 @@ check_domain(Domain) ->
transportDomainUdpIpv4,
transportDomainUdpIpv6
],
- AllDomains =
- [
- transportDomainUdpIpv4,
- transportDomainUdpIpv6,
- transportDomainUdpIpv4z,
- transportDomainUdpIpv6z,
- transportDomainTcpIpv4,
- transportDomainTcpIpv6,
- transportDomainTcpIpv4z,
- transportDomainTcpIpv6z,
- transportDomainSctpIpv4,
- transportDomainSctpIpv6,
- transportDomainSctpIpv4z,
- transportDomainSctpIpv6z,
- transportDomainLocal,
- transportDomainUdpDns,
- transportDomainTcpDns,
- transportDomainSctpDns
- ],
+ AllDomains = all_domains(),
case lists:member(Domain, SupportedDomains) of
true ->
ok;
diff --git a/lib/snmp/src/misc/snmp_config.erl b/lib/snmp/src/misc/snmp_config.erl
index fcbc6a88c9..6ab20e3e48 100644
--- a/lib/snmp/src/misc/snmp_config.erl
+++ b/lib/snmp/src/misc/snmp_config.erl
@@ -337,7 +337,7 @@ config_agent_sys() ->
{dir, ATLDir},
{size, ATLSize},
{repair, ATLRepair},
- {seqno, ATLSeqNo}]}];
+ {seqno, ATLSeqNo}]}];
no ->
[]
end,
@@ -568,7 +568,7 @@ config_agent_snmp(Dir, Vsns) ->
false ->
ok
end,
- i("The following agent files were written: agent.conf, "
+ i("The following agent files where written: agent.conf, "
"community.conf,~n"
"standard.conf, target_addr.conf, "
"target_params.conf, ~n"
@@ -776,7 +776,7 @@ config_manager_snmp(Dir, Vsns) ->
Users, Agents, Usms)) of
ok ->
i("~n- - - - - - - - - - - - -"),
- i("The following manager files were written: "
+ i("The following manager files where written: "
"manager.conf, agents.conf " ++
case lists:member(v3, Vsns) of
true ->
@@ -2350,7 +2350,9 @@ write_sys_config_file_manager_atl_opt(Fid, {type, Type}) ->
write_sys_config_file_manager_atl_opt(Fid, {size, Size}) ->
ok = io:format(Fid, "{size, ~w}", [Size]);
write_sys_config_file_manager_atl_opt(Fid, {repair, Rep}) ->
- ok = io:format(Fid, "{repair, ~w}", [Rep]).
+ ok = io:format(Fid, "{repair, ~w}", [Rep]);
+write_sys_config_file_manager_atl_opt(Fid, {seqno, SeqNo}) ->
+ ok = io:format(Fid, "{seqno, ~w}", [SeqNo]).
header() ->
diff --git a/lib/snmp/test/snmp_compiler_test.erl b/lib/snmp/test/snmp_compiler_test.erl
index 2e6020ae7a..cee11ba97a 100644
--- a/lib/snmp/test/snmp_compiler_test.erl
+++ b/lib/snmp/test/snmp_compiler_test.erl
@@ -47,6 +47,7 @@
module_identity/1,
agent_capabilities/1,
module_compliance/1,
+ warnings_as_errors/1,
otp_6150/1,
otp_8574/1,
@@ -97,9 +98,10 @@ all() ->
description,
oid_conflicts,
imports,
- module_identity,
- agent_capabilities,
- module_compliance,
+ module_identity,
+ agent_capabilities,
+ module_compliance,
+ warnings_as_errors,
{group, tickets}
].
@@ -152,6 +154,8 @@ description(Config) when is_list(Config) ->
ok.
+%%======================================================================
+
oid_conflicts(suite) -> [];
oid_conflicts(Config) when is_list(Config) ->
put(tname,oid_conflicts),
@@ -165,18 +169,24 @@ oid_conflicts(Config) when is_list(Config) ->
ok.
+%%======================================================================
+
imports(suite) ->
[];
imports(Config) when is_list(Config) ->
?SKIP(not_yet_implemented).
+%%======================================================================
+
module_identity(suite) ->
[];
module_identity(Config) when is_list(Config) ->
?SKIP(not_yet_implemented).
+%%======================================================================
+
agent_capabilities(suite) ->
[];
agent_capabilities(Config) when is_list(Config) ->
@@ -218,6 +228,8 @@ agent_capabilities(Config) when is_list(Config) ->
ok.
+%%======================================================================
+
module_compliance(suite) ->
[];
module_compliance(Config) when is_list(Config) ->
@@ -259,6 +271,31 @@ module_compliance(Config) when is_list(Config) ->
ok.
+%%======================================================================
+
+warnings_as_errors(suite) ->
+ ["OTP-9437"];
+warnings_as_errors(Config) when is_list(Config) ->
+ put(tname,warnings_as_errors),
+ p("starting with Config: ~p~n", [Config]),
+ Dir = ?config(comp_dir, Config),
+ MibDir = ?config(mib_dir, Config),
+ MibFile = join(MibDir, "OTP8574-MIB.mib"),
+ OutFile = join(Dir, "OTP8574-MIB.bin"),
+ Opts = [{group_check, false},
+ {outdir, Dir},
+ {verbosity, trace},
+ relaxed_row_name_assign_check],
+ {error, compilation_failed} =
+ snmpc:compile(MibFile, [warnings_as_errors|Opts]),
+ false = filelib:is_regular(OutFile),
+ {ok, _} = snmpc:compile(MibFile, Opts),
+ true = filelib:is_regular(OutFile),
+ ok.
+
+
+%%======================================================================
+
otp_6150(suite) ->
[];
otp_6150(Config) when is_list(Config) ->
@@ -273,6 +310,8 @@ otp_6150(Config) when is_list(Config) ->
ok.
+%%======================================================================
+
otp_8574(suite) ->
[];
otp_8574(Config) when is_list(Config) ->
@@ -304,6 +343,8 @@ otp_8574(Config) when is_list(Config) ->
end.
+%%======================================================================
+
otp_8595(suite) ->
[];
otp_8595(Config) when is_list(Config) ->
diff --git a/lib/snmp/test/snmp_manager_test.erl b/lib/snmp/test/snmp_manager_test.erl
index 0b536748fb..d18f20d359 100644
--- a/lib/snmp/test/snmp_manager_test.erl
+++ b/lib/snmp/test/snmp_manager_test.erl
@@ -61,6 +61,7 @@
register_agent1/1,
register_agent2/1,
+ register_agent3/1,
info/1,
@@ -383,12 +384,12 @@ end_per_testcase2(Case, Config) ->
all() ->
[
{group, start_and_stop_tests},
- {group, misc_tests},
+ {group, misc_tests},
{group, user_tests},
- {group, agent_tests},
+ {group, agent_tests},
{group, request_tests},
{group, event_tests},
- discovery,
+ discovery,
{group, tickets}
].
@@ -417,7 +418,8 @@ groups() ->
{agent_tests, [],
[
register_agent1,
- register_agent2
+ register_agent2,
+ register_agent3
]
},
{request_tests, [],
@@ -477,14 +479,14 @@ groups() ->
},
{event_tests, [],
[
- trap1,
- trap2,
- inform1,
- inform2,
- inform3,
- inform4,
- inform_swarm,
- report
+ trap1%% ,
+ %% trap2,
+ %% inform1,
+ %% inform2,
+ %% inform3,
+ %% inform4,
+ %% inform_swarm,
+ %% report
]
},
{tickets, [],
@@ -1134,6 +1136,7 @@ register_agent1(suite) ->
register_agent1(Config) when is_list(Config) ->
process_flag(trap_exit, true),
put(tname,ra1),
+
p("starting with Config: ~p~n", [Config]),
ManagerNode = start_manager_node(),
@@ -1164,7 +1167,7 @@ register_agent1(Config) when is_list(Config) ->
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
- p("register user(s) calvin & hobbe"),
+ p("register user(s) user_alfa & user_beta"),
?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []),
?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []),
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
@@ -1293,7 +1296,7 @@ register_agent2(Config) when is_list(Config) ->
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
- p("register user(s) calvin & hobbe"),
+ p("register user(s) user_alfa & user_beta"),
?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []),
?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []),
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
@@ -1348,7 +1351,7 @@ register_agent2(Config) when is_list(Config) ->
end,
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
-
+
p("unregister user user_alfa"),
?line ok = mgr_unregister_user(ManagerNode, user_alfa),
@@ -1377,7 +1380,157 @@ register_agent2(Config) when is_list(Config) ->
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
- p("unregister user hobbe"),
+ p("unregister user user_beta"),
+ ?line ok = mgr_unregister_user(ManagerNode, user_beta),
+
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ ?SLEEP(1000),
+
+ p("stop snmp application (with only manager)"),
+ ?line ok = stop_snmp(ManagerNode),
+
+ ?SLEEP(1000),
+
+ stop_node(ManagerNode),
+
+ ?SLEEP(1000),
+
+ p("end"),
+ ok.
+
+
+%%======================================================================
+
+register_agent3(doc) ->
+ ["Test registration of agents with the NEW interface functions "
+ "and specifying transport domain"];
+register_agent3(suite) ->
+ [];
+register_agent3(Config) when is_list(Config) ->
+ process_flag(trap_exit, true),
+ put(tname, ra3),
+ p("starting with Config: ~p~n", [Config]),
+
+ ManagerNode = start_manager_node(),
+
+ ConfDir = ?config(manager_conf_dir, Config),
+ DbDir = ?config(manager_db_dir, Config),
+ LocalHost = snmp_test_lib:localhost(),
+
+
+ write_manager_conf(ConfDir),
+
+ Opts = [{server, [{verbosity, trace}]},
+ {net_if, [{verbosity, trace}]},
+ {note_store, [{verbosity, trace}]},
+ {config, [{verbosity, trace}, {dir, ConfDir}, {db_dir, DbDir}]}],
+
+
+ p("load snmp application"),
+ ?line ok = load_snmp(ManagerNode),
+
+ p("set manager env for the snmp application"),
+ ?line ok = set_mgr_env(ManagerNode, Opts),
+
+ p("starting snmp application (with only manager)"),
+ ?line ok = start_snmp(ManagerNode),
+
+ p("started"),
+
+ ?SLEEP(1000),
+
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ p("register user(s) user_alfa & user_beta"),
+ ?line ok = mgr_register_user(ManagerNode, user_alfa, snmpm_user_default, []),
+ ?line ok = mgr_register_user(ManagerNode, user_beta, snmpm_user_default, []),
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ p("register agent(s)"),
+ TargetName1 = "agent2",
+ ?line ok = mgr_register_agent(ManagerNode, user_alfa, TargetName1,
+ [{tdomain, transportDomainUdpIpv4},
+ {address, LocalHost},
+ {port, 5001},
+ {engine_id, "agentEngineId-1"}]),
+ TargetName2 = "agent3",
+ ?line ok = mgr_register_agent(ManagerNode, user_alfa, TargetName2,
+ [{tdomain, transportDomainUdpIpv6},
+ {address, LocalHost},
+ {port, 5002},
+ {engine_id, "agentEngineId-2"}]),
+ TargetName3 = "agent4",
+ ?line {error, {unsupported_domain, _} = Reason4} =
+ mgr_register_agent(ManagerNode, user_beta, TargetName3,
+ [{tdomain, transportDomainTcpIpv4},
+ {address, LocalHost},
+ {port, 5003},
+ {engine_id, "agentEngineId-3"}]),
+ p("Expected registration failure: ~p", [Reason4]),
+ TargetName4 = "agent5",
+ ?line {error, {unknown_domain, _} = Reason5} =
+ mgr_register_agent(ManagerNode, user_beta, TargetName4,
+ [{tdomain, transportDomainUdpIpv4_bad},
+ {address, LocalHost},
+ {port, 5004},
+ {engine_id, "agentEngineId-4"}]),
+ p("Expected registration failure: ~p", [Reason5]),
+
+ p("verify all agent(s): expect 2"),
+ case mgr_which_agents(ManagerNode) of
+ Agents1 when length(Agents1) =:= 2 ->
+ p("all agents: ~p~n", [Agents1]),
+ ok;
+ Agents1 ->
+ ?FAIL({agent_registration_failure, Agents1})
+ end,
+
+ p("verify user_alfa agent(s)"),
+ case mgr_which_agents(ManagerNode, user_alfa) of
+ Agents2 when length(Agents2) =:= 2 ->
+ p("calvin agents: ~p~n", [Agents2]),
+ ok;
+ Agents2 ->
+ ?FAIL({agent_registration_failure, Agents2})
+ end,
+
+ p("verify user_beta agent(s)"),
+ case mgr_which_agents(ManagerNode, user_beta) of
+ Agents3 when length(Agents3) =:= 0 ->
+ p("hobbe agents: ~p~n", [Agents3]),
+ ok;
+ Agents3 ->
+ ?FAIL({agent_registration_failure, Agents3})
+ end,
+
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ p("unregister user user_alfa"),
+ ?line ok = mgr_unregister_user(ManagerNode, user_alfa),
+
+ p("verify all agent(s): expect 0"),
+ case mgr_which_agents(ManagerNode) of
+ Agents4 when length(Agents4) =:= 0 ->
+ p("all agents: ~p~n", [Agents4]),
+ ok;
+ Agents4 ->
+ ?FAIL({agent_unregistration_failure, Agents4})
+ end,
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ p("verify all agent(s): expect 0"),
+ case mgr_which_agents(ManagerNode) of
+ [] ->
+ ok;
+ Agents5 ->
+ p("all agents: ~p~n", [Agents5]),
+ ?FAIL({agent_unregistration_failure, Agents5})
+ end,
+
+ p("manager info: ~p~n", [mgr_info(ManagerNode)]),
+
+ p("unregister user user_beta"),
?line ok = mgr_unregister_user(ManagerNode, user_beta),
p("manager info: ~p~n", [mgr_info(ManagerNode)]),
diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk
index 29228fc59b..08251ab9ea 100644
--- a/lib/snmp/vsn.mk
+++ b/lib/snmp/vsn.mk
@@ -17,6 +17,6 @@
#
# %CopyrightEnd%
-SNMP_VSN = 4.20
+SNMP_VSN = 4.21
PRE_VSN =
APP_VSN = "snmp-$(SNMP_VSN)$(PRE_VSN)"
diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl
index d9f4a76d80..37a021e7cf 100644
--- a/lib/ssl/test/ssl_basic_SUITE.erl
+++ b/lib/ssl/test/ssl_basic_SUITE.erl
@@ -257,7 +257,7 @@ all() ->
%%different_ca_peer_sign,
no_reuses_session_server_restart_new_cert,
no_reuses_session_server_restart_new_cert_file, reuseaddr,
- hibernate
+ hibernate, connect_twice
].
groups() ->
@@ -3609,6 +3609,54 @@ hibernate(Config) ->
ssl_test_lib:close(Client).
%%--------------------------------------------------------------------
+
+connect_twice(doc) ->
+ [""];
+connect_twice(suite) ->
+ [];
+connect_twice(Config) when is_list(Config) ->
+ ClientOpts = ?config(client_opts, Config),
+ ServerOpts = ?config(server_opts, Config),
+
+ {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
+
+ Server =
+ ssl_test_lib:start_server([{node, ServerNode}, {port, 0},
+ {from, self()},
+ {mfa, {?MODULE, send_recv_result, []}},
+ {options, [{keepalive, true},{active, false}
+ | ServerOpts]}]),
+ Port = ssl_test_lib:inet_port(Server),
+ Client =
+ ssl_test_lib:start_client([{node, ClientNode}, {port, Port},
+ {host, Hostname},
+ {from, self()},
+ {mfa, {?MODULE, send_recv_result, []}},
+ {options, [{keepalive, true},{active, false}
+ | ClientOpts]}]),
+ Server ! listen,
+
+ {Client1, #sslsocket{}} =
+ ssl_test_lib:start_client([return_socket,
+ {node, ClientNode}, {port, Port},
+ {host, Hostname},
+ {from, self()},
+ {mfa, {?MODULE, send_recv_result, []}},
+ {options, [{keepalive, true},{active, false}
+ | ClientOpts]}]),
+
+ test_server:format("Testcase ~p, Client ~p Server ~p ~n",
+ [self(), Client, Server]),
+
+ ssl_test_lib:check_result(Server, ok, Client, ok),
+ ssl_test_lib:check_result(Server, ok, Client1, ok),
+
+ ssl_test_lib:close(Server),
+ ssl_test_lib:close(Client),
+ ssl_test_lib:close(Client1).
+
+
+%%--------------------------------------------------------------------
%%% Internal functions
%%--------------------------------------------------------------------
send_recv_result(Socket) ->
diff --git a/lib/ssl/test/ssl_session_cache_SUITE.erl b/lib/ssl/test/ssl_session_cache_SUITE.erl
index f80ac3c1a9..5ea45018e6 100644
--- a/lib/ssl/test/ssl_session_cache_SUITE.erl
+++ b/lib/ssl/test/ssl_session_cache_SUITE.erl
@@ -223,15 +223,14 @@ session_cleanup(Config)when is_list(Config) ->
%% Make sure session has expired and been cleaned up
check_timer(SessionTimer),
- test_server:sleep(?DELAY), %% Delay time + some extra time
+ test_server:sleep(?DELAY *2), %% Delay time + some extra time
- {status, _, _, StatusInfo1} = sys:get_status(whereis(ssl_manager)),
- [_, _,_, _, Prop1] = StatusInfo1,
- State1 = state(Prop1),
- DelayTimer = element(7, State1),
+ DelayTimer = get_delay_timer(),
check_timer(DelayTimer),
+ test_server:sleep(?SLEEP), %% Make sure clean has had to run
+
undefined = ssl_session_cache:lookup(Cache, {{Hostname, Port}, Id}),
undefined = ssl_session_cache:lookup(Cache, {Port, Id}),
@@ -253,6 +252,18 @@ check_timer(Timer) ->
test_server:sleep(Int),
check_timer(Timer)
end.
+
+get_delay_timer() ->
+ {status, _, _, StatusInfo} = sys:get_status(whereis(ssl_manager)),
+ [_, _,_, _, Prop] = StatusInfo,
+ State = state(Prop),
+ case element(7, State) of
+ undefined ->
+ test_server:sleep(?SLEEP),
+ get_delay_timer();
+ DelayTimer ->
+ DelayTimer
+ end.
%%--------------------------------------------------------------------
session_cache_process_list(doc) ->
["Test reuse of sessions (short handshake)"];
diff --git a/lib/stdlib/src/erl_internal.erl b/lib/stdlib/src/erl_internal.erl
index 478f05e792..0b9b8b8e17 100644
--- a/lib/stdlib/src/erl_internal.erl
+++ b/lib/stdlib/src/erl_internal.erl
@@ -263,7 +263,6 @@ bif(bit_size, 1) -> true;
bif(bitstring_to_list, 1) -> true;
bif(byte_size, 1) -> true;
bif(check_process_code, 2) -> true;
-bif(concat_binary, 1) -> true;
bif(date, 0) -> true;
bif(delete_module, 1) -> true;
bif(demonitor, 1) -> true;
@@ -405,7 +404,6 @@ old_bif(bit_size, 1) -> true;
old_bif(bitstring_to_list, 1) -> true;
old_bif(byte_size, 1) -> true;
old_bif(check_process_code, 2) -> true;
-old_bif(concat_binary, 1) -> true;
old_bif(date, 0) -> true;
old_bif(delete_module, 1) -> true;
old_bif(disconnect_node, 1) -> true;
diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl
index 718ca2e91a..10b2ed2e49 100644
--- a/lib/stdlib/src/erl_scan.erl
+++ b/lib/stdlib/src/erl_scan.erl
@@ -408,7 +408,12 @@ set_attr(line, {Line,Column}, Fun) when ?ALINE(Line), ?COLUMN(Column) ->
end;
set_attr(line=Tag, Attrs, Fun) when is_list(Attrs) ->
{line,Line} = lists:keyfind(Tag, 1, Attrs),
- lists:keyreplace(Tag, 1, Attrs, {line,Fun(Line)});
+ case lists:keyreplace(Tag, 1, Attrs, {line,Fun(Line)}) of
+ [{line,Ln}] when ?ALINE(Ln) ->
+ Ln;
+ As ->
+ As
+ end;
set_attr(T1, T2, T3) ->
erlang:error(badarg, [T1,T2,T3]).
diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl
index 54c7283abf..165e03e506 100644
--- a/lib/stdlib/src/io_lib.erl
+++ b/lib/stdlib/src/io_lib.erl
@@ -100,7 +100,7 @@ fwrite(Format, Args) ->
-spec fread(Format, String) -> Result when
Format :: string(),
String :: string(),
- Result :: {'ok', InputList :: chars(), LeftOverChars :: string()}
+ Result :: {'ok', InputList :: [term()], LeftOverChars :: string()}
| {'more', RestFormat :: string(),
Nchars :: non_neg_integer(),
InputStack :: chars()}
@@ -115,7 +115,7 @@ fread(Chars, Format) ->
Format :: string(),
Return :: {'more', Continuation1 :: continuation()}
| {'done', Result, LeftOverChars :: string()},
- Result :: {'ok', InputList :: chars()}
+ Result :: {'ok', InputList :: [term()]}
| 'eof'
| {'error', What :: term()}.
diff --git a/lib/stdlib/src/io_lib_fread.erl b/lib/stdlib/src/io_lib_fread.erl
index 52aa4d073c..ded1346097 100644
--- a/lib/stdlib/src/io_lib_fread.erl
+++ b/lib/stdlib/src/io_lib_fread.erl
@@ -24,6 +24,10 @@
-import(lists, [reverse/1,reverse/2]).
+-define(is_whitespace(C),
+ ((C) =:= $\s orelse (C) =:= $\t
+ orelse (C) =:= $\r orelse (C) =:= $\n)).
+
%%-----------------------------------------------------------------------
%% fread(Continuation, CharList, FormatString)
@@ -106,31 +110,27 @@ fread_line(Format0, Line, N0, Results0, More, Newline) ->
fread(Format, Line) ->
fread(Format, Line, 0, []).
-fread([$~|Format0], Line, N, Results) ->
+fread([$~|Format0]=AllFormat, Line, N, Results) ->
{Format,F,Sup,Unicode} = fread_field(Format0),
- fread1(Format, F, Sup, Unicode, Line, N, Results, Format0);
-fread([$\s|Format], Line, N, Results) ->
- fread_skip_white(Format, Line, N, Results);
-fread([$\t|Format], Line, N, Results) ->
- fread_skip_white(Format, Line, N, Results);
-fread([$\r|Format], Line, N, Results) ->
- fread_skip_white(Format, Line, N, Results);
-fread([$\n|Format], Line, N, Results) ->
+ fread1(Format, F, Sup, Unicode, Line, N, Results, AllFormat);
+fread([C|Format], Line, N, Results) when ?is_whitespace(C) ->
fread_skip_white(Format, Line, N, Results);
fread([C|Format], [C|Line], N, Results) ->
fread(Format, Line, N+1, Results);
fread([_F|_Format], [_C|_Line], _N, _Results) ->
fread_error(input);
+fread([_|_]=Format, [], N, Results) ->
+ {more,Format,N,Results};
+fread([_|_], eof, 0, []) ->
+ %% This is at start of input so no error.
+ eof;
+fread([_|_], eof, _N, _Results) ->
+ %% This is an error as there is no more input.
+ fread_error(input);
fread([], Line, _N, Results) ->
{ok,reverse(Results),Line}.
-fread_skip_white(Format, [$\s|Line], N, Results) ->
- fread_skip_white(Format, Line, N+1, Results);
-fread_skip_white(Format, [$\t|Line], N, Results) ->
- fread_skip_white(Format, Line, N+1, Results);
-fread_skip_white(Format, [$\r|Line], N, Results) ->
- fread_skip_white(Format, Line, N+1, Results);
-fread_skip_white(Format, [$\n|Line], N, Results) ->
+fread_skip_white(Format, [C|Line], N, Results) when ?is_whitespace(C) ->
fread_skip_white(Format, Line, N+1, Results);
fread_skip_white(Format, Line, N, Results) ->
fread(Format, Line, N, Results).
@@ -166,9 +166,9 @@ fread1([$l|Format], _F, Sup, _U, Line, N, Res, _AllFormat) ->
fread(Format, Line, N, fread_result(Sup, N, Res));
fread1(_Format, _F, _Sup, _U, [], N, Res, AllFormat) ->
%% Need more input here.
- {more,[$~|AllFormat],N,Res};
-fread1(_Format, _F, _Sup, _U, eof, _N, [], _AllFormat) ->
- %% This is at start of format string so no error.
+ {more,AllFormat,N,Res};
+fread1(_Format, _F, _Sup, _U, eof, 0, [], _AllFormat) ->
+ %% This is at start of input so no error.
eof;
fread1(_Format, _F, _Sup, _U, eof, _N, _Res, _AllFormat) ->
%% This is an error as there is no more input.
@@ -386,26 +386,16 @@ fread_string_cs(Line0, N0, true) ->
%% fread_digits(Line, N, Base, Characters)
%% Read segments of things, return "thing" characters in reverse order.
-fread_skip_white([$\s|Line]) -> fread_skip_white(Line);
-fread_skip_white([$\t|Line]) -> fread_skip_white(Line);
-fread_skip_white([$\r|Line]) -> fread_skip_white(Line);
-fread_skip_white([$\n|Line]) -> fread_skip_white(Line);
+fread_skip_white([C|Line]) when ?is_whitespace(C) ->
+ fread_skip_white(Line);
fread_skip_white(Line) -> Line.
-fread_skip_white([$\s|Line], N) ->
- fread_skip_white(Line, N+1);
-fread_skip_white([$\t|Line], N) ->
- fread_skip_white(Line, N+1);
-fread_skip_white([$\r|Line], N) ->
- fread_skip_white(Line, N+1);
-fread_skip_white([$\n|Line], N) ->
+fread_skip_white([C|Line], N) when ?is_whitespace(C) ->
fread_skip_white(Line, N+1);
fread_skip_white(Line, N) -> {Line,N}.
-fread_skip_latin1_nonwhite([$\s|Line], N, Cs) -> {[$\s|Line],N,Cs};
-fread_skip_latin1_nonwhite([$\t|Line], N, Cs) -> {[$\t|Line],N,Cs};
-fread_skip_latin1_nonwhite([$\r|Line], N, Cs) -> {[$\r|Line],N,Cs};
-fread_skip_latin1_nonwhite([$\n|Line], N, Cs) -> {[$\n|Line],N,Cs};
+fread_skip_latin1_nonwhite([C|Line], N, Cs) when ?is_whitespace(C) ->
+ {[C|Line],N,Cs};
fread_skip_latin1_nonwhite([C|Line], N, []) when C > 255 ->
{[C|Line],N,error};
fread_skip_latin1_nonwhite([C|Line], N, Cs) when C > 255 ->
@@ -414,10 +404,8 @@ fread_skip_latin1_nonwhite([C|Line], N, Cs) ->
fread_skip_latin1_nonwhite(Line, N+1, [C|Cs]);
fread_skip_latin1_nonwhite([], N, Cs) -> {[],N,Cs}.
-fread_skip_nonwhite([$\s|Line], N, Cs) -> {[$\s|Line],N,Cs};
-fread_skip_nonwhite([$\t|Line], N, Cs) -> {[$\t|Line],N,Cs};
-fread_skip_nonwhite([$\r|Line], N, Cs) -> {[$\r|Line],N,Cs};
-fread_skip_nonwhite([$\n|Line], N, Cs) -> {[$\n|Line],N,Cs};
+fread_skip_nonwhite([C|Line], N, Cs) when ?is_whitespace(C) ->
+ {[C|Line],N,Cs};
fread_skip_nonwhite([C|Line], N, Cs) ->
fread_skip_nonwhite(Line, N+1, [C|Cs]);
fread_skip_nonwhite([], N, Cs) -> {[],N,Cs}.
diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl
index 39d017d430..db46670f61 100644
--- a/lib/stdlib/src/otp_internal.erl
+++ b/lib/stdlib/src/otp_internal.erl
@@ -449,7 +449,7 @@ obsolete_1(ssl_pkix, decode_cert, A) when A =:= 1; A =:= 2 ->
%% Added in R13B04.
obsolete_1(erlang, concat_binary, 1) ->
- {deprecated,{erlang,list_to_binary,1},"R15B"};
+ {removed,{erlang,list_to_binary,1},"R15B"};
%% Added in R14A.
obsolete_1(ssl, peercert, 2) ->
@@ -461,6 +461,10 @@ obsolete_1(public_key, pem_to_der, 1) ->
obsolete_1(public_key, decode_private_key, A) when A =:= 1; A =:= 2 ->
{deprecated,{public_key,pem_entry_decode,1},"R15A"};
+%% Added in R15B
+obsolete_1(asn1rt, F, _) when F == load_driver; F == unload_driver ->
+ {deprecated,"deprecated (will be removed in R16A); has no effect as drivers are no longer used."};
+
obsolete_1(_, _, _) ->
no.
diff --git a/lib/stdlib/src/queue.erl b/lib/stdlib/src/queue.erl
index 4c6b4d710b..afe917b151 100644
--- a/lib/stdlib/src/queue.erl
+++ b/lib/stdlib/src/queue.erl
@@ -56,16 +56,14 @@
new() -> {[],[]}. %{RearList,FrontList}
%% O(1)
--spec is_queue(Term) -> boolean() when
- Term :: term().
+-spec is_queue(Term :: term()) -> boolean().
is_queue({R,F}) when is_list(R), is_list(F) ->
true;
is_queue(_) ->
false.
%% O(1)
--spec is_empty(Q) -> boolean() when
- Q :: queue().
+-spec is_empty(Q :: queue()) -> boolean().
is_empty({[],[]}) ->
true;
is_empty({In,Out}) when is_list(In), is_list(Out) ->
@@ -74,16 +72,14 @@ is_empty(Q) ->
erlang:error(badarg, [Q]).
%% O(len(Q))
--spec len(Q) -> non_neg_integer() when
- Q :: queue().
+-spec len(Q :: queue()) -> non_neg_integer().
len({R,F}) when is_list(R), is_list(F) ->
length(R)+length(F);
len(Q) ->
erlang:error(badarg, [Q]).
%% O(len(Q))
--spec to_list(Q) -> list() when
- Q :: queue().
+-spec to_list(Q :: queue()) -> list().
to_list({In,Out}) when is_list(In), is_list(Out) ->
Out++lists:reverse(In, []);
to_list(Q) ->
@@ -92,8 +88,7 @@ to_list(Q) ->
%% Create queue from list
%%
%% O(length(L))
--spec from_list(L) -> queue() when
- L :: list().
+-spec from_list(L :: list()) -> queue().
from_list(L) when is_list(L) ->
f2r(L);
from_list(L) ->
@@ -102,9 +97,7 @@ from_list(L) ->
%% Return true or false depending on if element is in queue
%%
%% O(length(Q)) worst case
--spec member(Item, Q) -> boolean() when
- Item :: term(),
- Q :: queue().
+-spec member(Item :: term(), Q :: queue()) -> boolean().
member(X, {R,F}) when is_list(R), is_list(F) ->
lists:member(X, R) orelse lists:member(X, F);
member(X, Q) ->
@@ -117,10 +110,7 @@ member(X, Q) ->
%% Put at least one element in each list, if it is cheap
%%
%% O(1)
--spec in(Item, Q1) -> Q2 when
- Item :: term(),
- Q1 :: queue(),
- Q2 :: queue().
+-spec in(Item :: term(), Q1 :: queue()) -> Q2 :: queue().
in(X, {[_]=In,[]}) ->
{[X], In};
in(X, {In,Out}) when is_list(In), is_list(Out) ->
@@ -132,10 +122,7 @@ in(X, Q) ->
%% Put at least one element in each list, if it is cheap
%%
%% O(1)
--spec in_r(Item, Q1) -> Q2 when
- Item :: term(),
- Q1 :: queue(),
- Q2 :: queue().
+-spec in_r(Item :: term(), Q1 :: queue()) -> Q2 :: queue().
in_r(X, {[],[_]=F}) ->
{F,[X]};
in_r(X, {R,F}) when is_list(R), is_list(F) ->
@@ -146,10 +133,9 @@ in_r(X, Q) ->
%% Take from head/front
%%
%% O(1) amortized, O(len(Q)) worst case
--spec out(Q1) -> Result when
- Q1 :: queue(),
- Q2 :: queue(),
- Result :: {{value, Item :: term()}, Q2} | {empty, Q1}.
+-spec out(Q1 :: queue()) ->
+ {{value, Item :: term()}, Q2 :: queue()} |
+ {empty, Q1 :: queue()}.
out({[],[]}=Q) ->
{empty,Q};
out({[V],[]}) ->
@@ -167,10 +153,9 @@ out(Q) ->
%% Take from tail/rear
%%
%% O(1) amortized, O(len(Q)) worst case
--spec out_r(Q1) -> Result when
- Q1 :: queue(),
- Q2 :: queue(),
- Result :: {{value, Item :: term()}, Q2} | {empty, Q1}.
+-spec out_r(Q1 :: queue()) ->
+ {{value, Item :: term()}, Q2 :: queue()} |
+ {empty, Q1 :: queue()}.
out_r({[],[]}=Q) ->
{empty,Q};
out_r({[],[V]}) ->
@@ -191,9 +176,7 @@ out_r(Q) ->
%% Return the first element in the queue
%%
%% O(1) since the queue is supposed to be well formed
--spec get(Q) -> Item when
- Q :: queue(),
- Item :: term().
+-spec get(Q :: queue()) -> Item :: term().
get({[],[]}=Q) ->
erlang:error(empty, [Q]);
get({R,F}) when is_list(R), is_list(F) ->
@@ -212,9 +195,7 @@ get([_|R], []) -> % malformed queue -> O(len(Q))
%% Return the last element in the queue
%%
%% O(1) since the queue is supposed to be well formed
--spec get_r(Q) -> Item when
- Q :: queue(),
- Item :: term().
+-spec get_r(Q :: queue()) -> Item :: term().
get_r({[],[]}=Q) ->
erlang:error(empty, [Q]);
get_r({[H|_],F}) when is_list(F) ->
@@ -229,9 +210,7 @@ get_r(Q) ->
%% Return the first element in the queue
%%
%% O(1) since the queue is supposed to be well formed
--spec peek(Q) -> 'empty' | {'value',Item} when
- Q :: queue(),
- Item :: term().
+-spec peek(Q :: queue()) -> empty | {value,Item :: term()}.
peek({[],[]}) ->
empty;
peek({R,[H|_]}) when is_list(R) ->
@@ -246,9 +225,7 @@ peek(Q) ->
%% Return the last element in the queue
%%
%% O(1) since the queue is supposed to be well formed
--spec peek_r(Q) -> 'empty' | {'value',Item} when
- Q :: queue(),
- Item :: term().
+-spec peek_r(Q :: queue()) -> empty | {value,Item :: term()}.
peek_r({[],[]}) ->
empty;
peek_r({[H|_],F}) when is_list(F) ->
@@ -263,9 +240,7 @@ peek_r(Q) ->
%% Remove the first element and return resulting queue
%%
%% O(1) amortized
--spec drop(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec drop(Q1 :: queue()) -> Q2 :: queue().
drop({[],[]}=Q) ->
erlang:error(empty, [Q]);
drop({[_],[]}) ->
@@ -283,9 +258,7 @@ drop(Q) ->
%% Remove the last element and return resulting queue
%%
%% O(1) amortized
--spec drop_r(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec drop_r(Q1 :: queue()) -> Q2 :: queue().
drop_r({[],[]}=Q) ->
erlang:error(empty, [Q]);
drop_r({[],[_]}) ->
@@ -306,9 +279,7 @@ drop_r(Q) ->
%% Return reversed queue
%%
%% O(1)
--spec reverse(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec reverse(Q1 :: queue()) -> Q2 :: queue().
reverse({R,F}) when is_list(R), is_list(F) ->
{F,R};
reverse(Q) ->
@@ -318,10 +289,7 @@ reverse(Q) ->
%%
%% Q2 empty: O(1)
%% else: O(len(Q1))
--spec join(Q1, Q2) -> Q3 when
- Q1 :: queue(),
- Q2 :: queue(),
- Q3 :: queue().
+-spec join(Q1 :: queue(), Q2 :: queue()) -> Q3 :: queue().
join({R,F}=Q, {[],[]}) when is_list(R), is_list(F) ->
Q;
join({[],[]}, {R,F}=Q) when is_list(R), is_list(F) ->
@@ -335,11 +303,8 @@ join(Q1, Q2) ->
%%
%% N = 0..len(Q)
%% O(max(N, len(Q)))
--spec split(N, Q1) -> {Q2,Q3} when
- N :: non_neg_integer(),
- Q1 :: queue(),
- Q2 :: queue(),
- Q3 :: queue().
+-spec split(N :: non_neg_integer(), Q1 :: queue()) ->
+ {Q2 :: queue(),Q3 :: queue()}.
split(0, {R,F}=Q) when is_list(R), is_list(F) ->
{{[],[]},Q};
split(N, {R,F}=Q) when is_integer(N), N >= 1, is_list(R), is_list(F) ->
@@ -380,10 +345,8 @@ split_r1_to_f2(N, [X|R1], F1, R2, F2) ->
%%
%% Fun(_) -> List: O(length(List) * len(Q))
%% else: O(len(Q)
--spec filter(Fun, Q1) -> Q2 when
- Fun :: fun((Item :: term()) -> boolean() | list()),
- Q1 :: queue(),
- Q2 :: queue().
+-spec filter(Fun, Q1 :: queue()) -> Q2 :: queue() when
+ Fun :: fun((Item :: term()) -> boolean() | list()).
filter(Fun, {R0,F0}) when is_function(Fun, 1), is_list(R0), is_list(F0) ->
F = filter_f(Fun, F0),
R = filter_r(Fun, R0),
@@ -459,10 +422,7 @@ filter_r(Fun, [X|R0]) ->
%% Cons to head
%%
--spec cons(Item, Q1) -> Q2 when
- Item :: term(),
- Q1 :: queue(),
- Q2 :: queue().
+-spec cons(Item :: term(), Q1 :: queue()) -> Q2 :: queue().
cons(X, Q) ->
in_r(X, Q).
@@ -471,9 +431,7 @@ cons(X, Q) ->
%% Return the first element in the queue
%%
%% O(1) since the queue is supposed to be well formed
--spec head(Q) -> Item when
- Q :: queue(),
- Item :: term().
+-spec head(Q :: queue()) -> Item :: term().
head({[],[]}=Q) ->
erlang:error(empty, [Q]);
head({R,F}) when is_list(R), is_list(F) ->
@@ -483,9 +441,7 @@ head(Q) ->
%% Remove head element and return resulting queue
%%
--spec tail(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec tail(Q1 :: queue()) -> Q2 :: queue().
tail(Q) ->
drop(Q).
@@ -493,35 +449,22 @@ tail(Q) ->
%% Cons to tail
%%
--spec snoc(Q1, Item) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue(),
- Item :: term().
+-spec snoc(Q1 :: queue(), Item :: term()) -> Q2 :: queue().
snoc(Q, X) ->
in(X, Q).
%% Return last element
--spec daeh(Q) -> Item when
- Q :: queue(),
- Item :: term().
+-spec daeh(Q :: queue()) -> Item :: term().
daeh(Q) -> get_r(Q).
--spec last(Q) -> Item when
- Q :: queue(),
- Item :: term().
+-spec last(Q :: queue()) -> Item :: term().
last(Q) -> get_r(Q).
%% Remove last element and return resulting queue
--spec liat(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec liat(Q1 :: queue()) -> Q2 :: queue().
liat(Q) -> drop_r(Q).
--spec lait(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec lait(Q1 :: queue()) -> Q2 :: queue().
lait(Q) -> drop_r(Q). %% Oops, mis-spelled 'tail' reversed. Forget this one.
--spec init(Q1) -> Q2 when
- Q1 :: queue(),
- Q2 :: queue().
+-spec init(Q1 :: queue()) -> Q2 :: queue().
init(Q) -> drop_r(Q).
%%--------------------------------------------------------------------------
diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl
index 31a4f94294..4298b2c701 100644
--- a/lib/stdlib/test/erl_scan_SUITE.erl
+++ b/lib/stdlib/test/erl_scan_SUITE.erl
@@ -737,6 +737,10 @@ set_attribute() ->
(catch {foo, erl_scan:set_attribute(line, [], F2)}), % type error
?line {'EXIT',{badarg,_}} =
(catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error
+
+ %% OTP-9412
+ ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}],
+ fun({nos,_V,VL}) -> VL end),
ok.
column_errors() ->
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 9d348b5f1a..9341300f90 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -72,6 +72,7 @@
exit_many_many_tables_owner/1]).
-export([write_concurrency/1, heir/1, give_away/1, setopts/1]).
-export([bad_table/1, types/1]).
+-export([otp_9423/1]).
-export([init_per_testcase/2, end_per_testcase/2]).
%% Convenience for manual testing
@@ -143,7 +144,8 @@ all() ->
otp_8166, exit_large_table_owner,
exit_many_large_table_owner, exit_many_tables_owner,
exit_many_many_tables_owner, write_concurrency, heir,
- give_away, setopts, bad_table, types].
+ give_away, setopts, bad_table, types,
+ otp_9423].
groups() ->
[{new, [],
@@ -5420,7 +5422,39 @@ types_do(Opts) ->
?line verify_etsmem(EtsMem).
-
+otp_9423(doc) -> ["vm-deadlock caused by race between ets:delete and others on write_concurrency table"];
+otp_9423(Config) when is_list(Config) ->
+ InitF = fun(_) -> {0,0} end,
+ ExecF = fun({S,F}) ->
+ receive
+ stop ->
+ io:format("~p got stop\n", [self()]),
+ [end_of_work | {"Succeded=",S,"Failed=",F}]
+ after 0 ->
+ %%io:format("~p (~p) doing lookup\n", [self(), {S,F}]),
+ try ets:lookup(otp_9423, key) of
+ [] -> {S+1,F}
+ catch
+ error:badarg -> {S,F+1}
+ end
+ end
+ end,
+ FiniF = fun(R) -> R end,
+ case run_workers(InitF, ExecF, FiniF, infinite, 1) of
+ Pids when is_list(Pids) ->
+ %%[P ! start || P <- Pids],
+ repeat(fun() -> ets:new(otp_9423, [named_table, public, {write_concurrency,true}]),
+ ets:delete(otp_9423)
+ end, 10000),
+ [P ! stop || P <- Pids],
+ wait_pids(Pids),
+ ok;
+
+ Skipped -> Skipped
+ end.
+
+
+
%
% Utility functions:
@@ -5434,21 +5468,30 @@ add_lists([E1|T1], [E2|T2], Acc) ->
add_lists(T1, T2, [E1+E2 | Acc]).
run_workers(InitF,ExecF,FiniF,Laps) ->
+ run_workers(InitF,ExecF,FiniF,Laps, 0).
+run_workers(InitF,ExecF,FiniF,Laps, Exclude) ->
case erlang:system_info(smp_support) of
true ->
- run_workers_do(InitF,ExecF,FiniF,Laps);
+ run_workers_do(InitF,ExecF,FiniF,Laps, Exclude);
false ->
{skipped,"No smp support"}
end.
-
+
run_workers_do(InitF,ExecF,FiniF,Laps) ->
- NumOfProcs = erlang:system_info(schedulers),
+ run_workers_do(InitF,ExecF,FiniF,Laps, 0).
+run_workers_do(InitF,ExecF,FiniF,Laps, Exclude) ->
+ ?line NumOfProcs = case erlang:system_info(schedulers) of
+ N when (N > Exclude) -> N - Exclude
+ end,
io:format("smp starting ~p workers\n",[NumOfProcs]),
Seeds = [{ProcN,random:uniform(9999)} || ProcN <- lists:seq(1,NumOfProcs)],
Parent = self(),
Pids = [spawn_link(fun()-> worker(Seed,InitF,ExecF,FiniF,Laps,Parent,NumOfProcs) end)
|| Seed <- Seeds],
- wait_pids(Pids).
+ case Laps of
+ infinite -> Pids;
+ _ -> wait_pids(Pids)
+ end.
worker({ProcN,Seed}, InitF, ExecF, FiniF, Laps, Parent, NumOfProcs) ->
io:format("smp worker ~p, seed=~p~n",[self(),Seed]),
@@ -5463,6 +5506,8 @@ worker_loop(0, _, State) ->
State;
worker_loop(_, _, [end_of_work|State]) ->
State;
+worker_loop(infinite, ExecF, State) ->
+ worker_loop(infinite,ExecF,ExecF(State));
worker_loop(N, ExecF, State) ->
worker_loop(N-1,ExecF,ExecF(State)).
@@ -5517,20 +5562,21 @@ etsmem() ->
case erlang:system_info({allocator,ets_alloc}) of
false -> undefined;
MemInfo ->
- MSBCS = lists:foldl(
- fun ({instance, _, L}, Acc) ->
- {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
- {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
- [MBCS,SBCS | Acc]
- end,
- [],
- MemInfo),
+ CS = lists:foldl(
+ fun ({instance, _, L}, Acc) ->
+ {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L),
+ {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
+ {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
+ [SBMBCS,MBCS,SBCS | Acc]
+ end,
+ [],
+ MemInfo),
lists:foldl(
fun(L, {Bl0,BlSz0}) ->
{value,{_,Bl,_,_}} = lists:keysearch(blocks, 1, L),
{value,{_,BlSz,_,_}} = lists:keysearch(blocks_size, 1, L),
{Bl0+Bl,BlSz0+BlSz}
- end, {0,0}, MSBCS)
+ end, {0,0}, CS)
end},
{Mem,AllTabs}.
diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl
index 54a98985cd..bb02a879c2 100644
--- a/lib/stdlib/test/io_SUITE.erl
+++ b/lib/stdlib/test/io_SUITE.erl
@@ -27,7 +27,7 @@
otp_6282/1, otp_6354/1, otp_6495/1, otp_6517/1, otp_6502/1,
manpage/1, otp_6708/1, otp_7084/1, otp_7421/1,
io_lib_collect_line_3_wb/1, cr_whitespace_in_string/1,
- io_fread_newlines/1, otp_8989/1]).
+ io_fread_newlines/1, otp_8989/1, io_lib_fread_literal/1]).
%-define(debug, true).
@@ -62,7 +62,7 @@ all() ->
otp_6282, otp_6354, otp_6495, otp_6517, otp_6502,
manpage, otp_6708, otp_7084, otp_7421,
io_lib_collect_line_3_wb, cr_whitespace_in_string,
- io_fread_newlines, otp_8989].
+ io_fread_newlines, otp_8989, io_lib_fread_literal].
groups() ->
[].
@@ -1995,3 +1995,29 @@ otp_8989(Suite) when is_list(Suite) ->
?line "Hel " = fmt("~-4.*s", [3,Hello]),
?line "Hel " = fmt("~*.*s", [-4,3,Hello]),
ok.
+
+io_lib_fread_literal(doc) ->
+ "OTP-9439 io_lib:fread bug for literate at end";
+io_lib_fread_literal(Suite) when is_list(Suite) ->
+ ?line {more,"~d",0,""} = io_lib:fread("~d", ""),
+ ?line {error,{fread,integer}} = io_lib:fread("~d", " "),
+ ?line {more,"~d",1,""} = io_lib:fread(" ~d", " "),
+ ?line {ok,[17],"X"} = io_lib:fread(" ~d", " 17X"),
+ %%
+ ?line {more,"d",0,""} = io_lib:fread("d", ""),
+ ?line {error,{fread,input}} = io_lib:fread("d", " "),
+ ?line {more,"d",1,""} = io_lib:fread(" d", " "),
+ ?line {ok,[],"X"} = io_lib:fread(" d", " dX"),
+ %%
+ ?line {done,eof,_} = io_lib:fread([], eof, "~d"),
+ ?line {done,eof,_} = io_lib:fread([], eof, " ~d"),
+ ?line {more,C1} = io_lib:fread([], " \n", " ~d"),
+ ?line {done,{error,{fread,input}},_} = io_lib:fread(C1, eof, " ~d"),
+ ?line {done,{ok,[18]},""} = io_lib:fread(C1, "18\n", " ~d"),
+ %%
+ ?line {done,eof,_} = io_lib:fread([], eof, "d"),
+ ?line {done,eof,_} = io_lib:fread([], eof, " d"),
+ ?line {more,C2} = io_lib:fread([], " \n", " d"),
+ ?line {done,{error,{fread,input}},_} = io_lib:fread(C2, eof, " d"),
+ ?line {done,{ok,[]},[]} = io_lib:fread(C2, "d\n", " d"),
+ ok.
diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl
index 969916abed..7a75114cb6 100644
--- a/lib/stdlib/test/supervisor_SUITE.erl
+++ b/lib/stdlib/test/supervisor_SUITE.erl
@@ -119,10 +119,9 @@ end_per_group(_GroupName, Config) ->
Config.
init_per_testcase(count_children_memory, Config) ->
- MemoryState = erlang:system_info(allocator),
- case count_children_allocator_test(MemoryState) of
- true -> Config;
- false ->
+ try erlang:memory() of
+ _ -> Config
+ catch error:notsup ->
{skip, "+Meamin used during test; erlang:memory/1 not available"}
end;
init_per_testcase(_Case, Config) ->
@@ -1118,17 +1117,6 @@ count_children_memory(Config) when is_list(Config) ->
[terminate(SupPid, Pid, child, kill) || {undefined, Pid, worker, _Modules} <- Children3],
[1,0,0,0] = get_child_counts(sup_test).
-count_children_allocator_test(MemoryState) ->
- Allocators = [temp_alloc, eheap_alloc, binary_alloc, ets_alloc,
- driver_alloc, sl_alloc, ll_alloc, fix_alloc, std_alloc,
- sys_alloc],
- MemoryStateList = element(4, MemoryState),
- AllocTypes = [lists:keyfind(Alloc, 1, MemoryStateList)
- || Alloc <- Allocators],
- AllocStates = [lists:keyfind(e, 1, AllocValue)
- || {_Type, AllocValue} <- AllocTypes],
- lists:all(fun(State) -> State == {e, true} end, AllocStates).
-
%%-------------------------------------------------------------------------
do_not_save_start_parameters_for_temporary_children(doc) ->
["Temporary children shall not be restarted so they should not "
diff --git a/lib/tools/doc/src/xref.xml b/lib/tools/doc/src/xref.xml
index 75ffa25311..17de66bb22 100644
--- a/lib/tools/doc/src/xref.xml
+++ b/lib/tools/doc/src/xref.xml
@@ -4,7 +4,7 @@
<erlref>
<header>
<copyright>
- <year>2000</year><year>2010</year>
+ <year>2000</year><year>2011</year>
<holder>Ericsson AB. All Rights Reserved.</holder>
</copyright>
<legalnotice>
@@ -1465,8 +1465,8 @@ Evaluates a predefined analysis.
<name>start(NameOrOptions) -> Return</name>
<fsummary>Create an Xref server.</fsummary>
<type>
- <v>Name = atom()()</v>
- <v>XrefOrOptions = Xref | Options</v>
+ <v>NameOrOptions = Name | Options</v>
+ <v>Name = atom()</v>
<v>Options = [Option] | Option</v>
<v>Option = {xref_mode, mode()} | term()</v>
<v>Return = {ok, pid()} | {error, {already_started, pid()}}</v>
@@ -1483,7 +1483,7 @@ Evaluates a predefined analysis.
<name>start(Name, Options) -> Return</name>
<fsummary>Create an Xref server.</fsummary>
<type>
- <v>Name = atom()()</v>
+ <v>Name = atom()</v>
<v>Options = [Option] | Option</v>
<v>Option = {xref_mode, mode()} | term()</v>
<v>Return = {ok, pid()} | {error, {already_started, pid()}}</v>