aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_threads.h
diff options
context:
space:
mode:
authorRickard Green <[email protected]>2012-02-20 12:14:26 +0100
committerRickard Green <[email protected]>2012-02-20 12:14:26 +0100
commit520ddbc83ec87bcec262680bd915184182e3998e (patch)
tree055f08a9a435b06f0e762e15954a9e309aef9651 /erts/emulator/beam/erl_threads.h
parent102c34e414f573081a93ca49162dd78cf91949b7 (diff)
parenta393026eb77ce06711c931bf68eed12d9f8ba9c9 (diff)
downloadotp-520ddbc83ec87bcec262680bd915184182e3998e.tar.gz
otp-520ddbc83ec87bcec262680bd915184182e3998e.tar.bz2
otp-520ddbc83ec87bcec262680bd915184182e3998e.zip
Merge branch 'rickard/barriers/OTP-9922' into maint
* rickard/barriers/OTP-9922: Reduce thread progress read operations in handle_aux_work() Misc memory barrier fixes
Diffstat (limited to 'erts/emulator/beam/erl_threads.h')
-rw-r--r--erts/emulator/beam/erl_threads.h319
1 files changed, 309 insertions, 10 deletions
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 065e7077c0..ee47c98009 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,235 @@
#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__
+/*
+ * --- Documentation of atomics and memory barriers --------------------------
+ *
+ * The following explicit memory barriers exist:
+ *
+ * - ERTS_THR_MEMORY_BARRIER
+ * Full memory barrier. Orders both loads, and stores. No
+ * load or store is allowed to be reordered over the
+ * barrier.
+ * - ERTS_THR_WRITE_MEMORY_BARRIER
+ * Write barrier. Orders *only* stores. These are not
+ * allowed to be reordered over the barrier.
+ * - ERTS_THR_READ_MEMORY_BARRIER
+ * Read barrier. Orders *only* loads. These are not
+ * allowed to be reordered over the barrier.
+ * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+ * Data dependency read barrier. Orders *only* loads
+ * according to data dependency across the barrier.
+ *
+ * If thread support has been disabled, these barriers will become no-ops.
+ *
+ * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
+ * be enabled only in the SMP enabled runtime system.
+ *
+ * --- Atomic operations ---
+ *
+ * Atomics operations exist for 32-bit, word size, and double word size
+ * integers. Function prototypes are listed below.
+ *
+ * Each function implementing an atomic operation exist with the following
+ * implied memory barrier semantics. Not all combinations are useful, but
+ * all of them exist for simplicity. <B> is suffix in function name:
+ *
+ * - <B> - Description
+ *
+ * - mb - Full memory barrier. Orders both loads, and
+ * stores before, and after the atomic operation.
+ * No load or store is allowed to be reordered
+ * over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ * stores appearing *before* the atomic
+ * operation. These are not allowed to be
+ * reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ * appearing *after* the atomic operation. These
+ * are not allowed to be reordered over the
+ * atomic operation.
+ * - wb - Write barrier. Orders *only* stores. These are
+ * not allowed to be reordered over the barrier.
+ * Store in atomic operation is ordered *after*
+ * the barrier.
+ * - rb - Read barrier. Orders *only* loads. These are
+ * not allowed to be reordered over the barrier.
+ * Load in atomic operation is ordered *before*
+ * the barrier.
+ * - ddrb - Data dependency read barrier. Orders *only*
+ * loads according to data dependency across the
+ * barrier. Load in atomic operation is ordered
+ * before the barrier.
+ *
+ * If thread support has been disabled, these functions are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply any memory barriers.
+ *
+ * If the atomic operations are prefixed with erts_smp_ instead of only
+ * erts_ the atomic operations will only be atomic in the SMP enabled
+ * runtime system, and will be mapped to non-atomic operations without
+ * memory barriers in the runtime system without SMP support. Atomic
+ * operations with erts_smp_ prefix should use the atomic types
+ * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
+ * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
+ * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
+ * are the same.
+ *
+ * --- 32-bit atomic operations ---
+ *
+ * The following 32-bit atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above). Note
+ * that sizeof(erts_atomic32_t) might be larger than 4!
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Set value:
+ * void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Read; returns current value:
+ * erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment; returns resulting value:
+ * erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ * erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment:
+ * void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement:
+ * void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
+ *
+ * Add value; returns resulting value:
+ * erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Add value:
+ * void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Bitwise-or; returns previous value:
+ * erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Bitwise-and; returns previous value:
+ * erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Exchange; returns previous value:
+ * erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ * erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
+ * erts_aint32_t new,
+ * erts_aint32_t exp);
+ *
+ * --- Word size atomic operations ---
+ *
+ * The following word size (same size as sizeof(void *)) atomic
+ * operations exist. <B> should be replaced with a supported
+ * memory barrier (see above). Note that sizeof(erts_atomic_t)
+ * might be larger than sizeof(void *)!
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_atomic_init_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Set value;
+ * void erts_atomic_set_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Read; returns current value:
+ * erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment; returns resulting value:
+ * erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ * erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment:
+ * void erts_atomic_inc_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement:
+ * void erts_atomic_dec_<B>(erts_atomic_t *atmc);
+ *
+ * Add value; returns resulting value:
+ * erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Add value:
+ * void erts_atomic_add_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Bitwise-or; returns previous value:
+ * erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Bitwise-and; returns previous value:
+ * erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Exchange; returns previous value:
+ * erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ * erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
+ * erts_aint_t new,
+ * erts_aint_t exp);
+ *
+ * --- Double word size atomic operations ---
+ *
+ * The following double word atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above).
+ *
+ * Note that sizeof(erts_dw_atomic_t) usually is larger than
+ * 2*sizeof(void *)!
+ *
+ * The erts_dw_aint_t data type should be accessed as if it was defined
+ * like this:
+ *
+ * typedef struct {
+ * erts_aint_t sint[2];
+ * } erts_dw_aint_t;
+ *
+ * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
+ * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Set; value is written into 'val':
+ * void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Read; value is written into 'val':
+ * void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Compare and exchange; returns a value != 0 if exchange was
+ * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
+ * contains the same value as in memory when the function is called,
+ * 'new' is written to memory; otherwise, not. If exchange was not
+ * made, 'exp_act' contains the actual value in memory:
+ * int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *new,
+ * erts_dw_aint_t *exp_act);
+ */
+
#define ERTS_SPIN_BODY ETHR_SPIN_BODY
#include "sys.h"
@@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t;
#endif
#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
#ifdef ERTS_ENABLE_LOCK_COUNT
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
@@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t;
#define erts_aint32_t ethr_sint32_t
#define erts_atomic32_t ethr_atomic32_t
+#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
+#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
+
/* spinlock */
typedef struct {
ethr_spinlock_t slck;
@@ -149,6 +384,9 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *);
#else /* #ifdef USE_THREADS */
#define ERTS_THR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
@@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
+#ifdef USE_THREADS
+
/*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If thread support has been disabled, they are mapped to
- * functions that performs the same operation, but aren't atomic
- * and don't imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of this file for info on atomics.
*/
-#ifdef USE_THREADS
-
/* Double word size atomics */
#define erts_dw_atomic_init_nob ethr_dw_atomic_init
@@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
+#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
+#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
+#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
+#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
+
#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
@@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+
#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
@@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
+#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
+#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
+#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
+#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
+#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
+#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
+#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
+#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
+#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
+#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
+#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
+#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+
#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
@@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+#define erts_atomic_init_ddrb erts_no_atomic_set
+#define erts_atomic_set_ddrb erts_no_atomic_set
+#define erts_atomic_read_ddrb erts_no_atomic_read
+#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_atomic_add_ddrb erts_no_atomic_add
+#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
@@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+#define erts_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read