/* * %CopyrightBegin% * * Copyright Ericsson AB 2001-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * %CopyrightEnd% */ /* Description: Error checking thread interface to the ethread library. * All functions terminates the emulator on failure. * Author: Rickard Green */ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ /* * --- Documentation of atomics and memory barriers -------------------------- * * The following explicit memory barriers exist: * * - ERTS_THR_MEMORY_BARRIER * Full memory barrier. Orders both loads, and stores. No * load or store is allowed to be reordered over the * barrier. * - ERTS_THR_WRITE_MEMORY_BARRIER * Write barrier. Orders *only* stores. These are not * allowed to be reordered over the barrier. * - ERTS_THR_READ_MEMORY_BARRIER * Read barrier. Orders *only* loads. These are not * allowed to be reordered over the barrier. * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER * Data dependency read barrier. Orders *only* loads * according to data dependency across the barrier. * * --- Atomic operations --- * * Atomics operations exist for 32-bit, word size, and double word size * integers. Function prototypes are listed below. * * Each function implementing an atomic operation exist with the following * implied memory barrier semantics. Not all combinations are useful, but * all of them exist for simplicity. is suffix in function name: * * - - Description * * - mb - Full memory barrier. Orders both loads, and * stores before, and after the atomic operation. * No load or store is allowed to be reordered * over the atomic operation. * - relb - Release barrier. Orders both loads, and * stores appearing *before* the atomic * operation. These are not allowed to be * reordered over the atomic operation. * - acqb - Acquire barrier. Orders both loads, and stores * appearing *after* the atomic operation. These * are not allowed to be reordered over the * atomic operation. * - wb - Write barrier. Orders *only* stores. These are * not allowed to be reordered over the barrier. * Store in atomic operation is ordered *after* * the barrier. * - rb - Read barrier. Orders *only* loads. These are * not allowed to be reordered over the barrier. * Load in atomic operation is ordered *before* * the barrier. * - ddrb - Data dependency read barrier. Orders *only* * loads according to data dependency across the * barrier. Load in atomic operation is ordered * before the barrier. * * --- 32-bit atomic operations --- * * The following 32-bit atomic operations exist. should be * replaced with a supported memory barrier (see above). Note * that sizeof(erts_atomic32_t) might be larger than 4! * * * Initialize (not necessarily the same as the set operation): * void erts_atomic32_init_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Set value: * void erts_atomic32_set_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Read; returns current value: * erts_aint32_t erts_atomic32_read_(erts_atomic32_t *atmc); * * Increment; returns resulting value: * erts_aint32_t erts_atomic32_inc_read_(erts_atomic32_t *atmc); * * Decrement; returns resulting value: * erts_aint32_t erts_atomic32_dec_read_(erts_atomic32_t *atmc); * * Increment: * void erts_atomic32_inc_(erts_atomic32_t *atmc); * * Decrement: * void erts_atomic32_dec_(erts_atomic32_t *atmc); * * Add value; returns resulting value: * erts_aint32_t erts_atomic32_add_read_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Add value: * void erts_atomic32_add_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Bitwise-or; returns previous value: * erts_aint32_t erts_atomic32_read_bor_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Bitwise-and; returns previous value: * erts_aint32_t erts_atomic32_read_band_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Exchange; returns previous value: * erts_aint32_t erts_atomic32_xchg_(erts_atomic32_t *atmc, * erts_aint32_t val); * * Compare and exchange; returns previous or current value. If * returned value equals 'exp' the value was changed to 'new'; * otherwise not: * erts_aint32_t erts_atomic32_cmpxchg_(erts_atomic32_t *a, * erts_aint32_t new, * erts_aint32_t exp); * * --- Word size atomic operations --- * * The following word size (same size as sizeof(void *)) atomic * operations exist. should be replaced with a supported * memory barrier (see above). Note that sizeof(erts_atomic_t) * might be larger than sizeof(void *)! * * Initialize (not necessarily the same as the set operation): * void erts_atomic_init_(erts_atomic_t *atmc, * erts_aint_t val); * * Set value; * void erts_atomic_set_(erts_atomic_t *atmc, * erts_aint_t val); * * Read; returns current value: * erts_aint_t erts_atomic_read_(erts_atomic_t *atmc); * * Increment; returns resulting value: * erts_aint_t erts_atomic_inc_read_(erts_atomic_t *atmc); * * Decrement; returns resulting value: * erts_aint_t erts_atomic_dec_read_(erts_atomic_t *atmc); * * Increment: * void erts_atomic_inc_(erts_atomic_t *atmc); * * Decrement: * void erts_atomic_dec_(erts_atomic_t *atmc); * * Add value; returns resulting value: * erts_aint_t erts_atomic_add_read_(erts_atomic_t *atmc, * erts_aint_t val); * * Add value: * void erts_atomic_add_(erts_atomic_t *atmc, * erts_aint_t val); * * Bitwise-or; returns previous value: * erts_aint_t erts_atomic_read_bor_(erts_atomic_t *atmc, * erts_aint_t val); * * Bitwise-and; returns previous value: * erts_aint_t erts_atomic_read_band_(erts_atomic_t *atmc, * erts_aint_t val); * * Exchange; returns previous value: * erts_aint_t erts_atomic_xchg_(erts_atomic_t *atmc, * erts_aint_t val); * * Compare and exchange; returns previous or current value. If * returned value equals 'exp' the value was changed to 'new'; * otherwise not: * erts_aint_t erts_atomic_cmpxchg_(erts_atomic_t *a, * erts_aint_t new, * erts_aint_t exp); * * --- Double word size atomic operations --- * * The following double word atomic operations exist. should be * replaced with a supported memory barrier (see above). * * Note that sizeof(erts_dw_atomic_t) usually is larger than * 2*sizeof(void *)! * * The erts_dw_aint_t data type should be accessed as if it was defined * like this: * * typedef struct { * erts_aint_t sint[2]; * } erts_dw_aint_t; * * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'. * * * Initialize (not necessarily the same as the set operation): * void erts_dw_atomic_init_(erts_dw_atomic_t *var, * erts_dw_aint_t *val); * * Set; value is written into 'val': * void erts_dw_atomic_set_(erts_dw_atomic_t *var, * erts_dw_aint_t *val); * * Read; value is written into 'val': * void erts_dw_atomic_read_(erts_dw_atomic_t *var, * erts_dw_aint_t *val); * * Compare and exchange; returns a value != 0 if exchange was * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act' * contains the same value as in memory when the function is called, * 'new' is written to memory; otherwise, not. If exchange was not * made, 'exp_act' contains the actual value in memory: * int erts_dw_atomic_cmpxchg_(erts_dw_atomic_t *var, * erts_dw_aint_t *new, * erts_dw_aint_t *exp_act); */ #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" #include "erl_lock_flags.h" #include "erl_term.h" #define ETHR_TRY_INLINE_FUNCS #include "ethread.h" #include "erl_lock_check.h" #include "erl_lock_count.h" #if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4 /* * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have * only seen this bug in glibc versions before 2.4. Note that condition * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by * this bug since these implementations may use mutexes internally. */ # define ERTS_THR_HAVE_BUSY_DESTROY_BUG #endif #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER #define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER #define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER #define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER #ifdef ERTS_ENABLE_LOCK_POSITION #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) #define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__) #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__) #define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__) #define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__) #define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__) #define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__) #define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__) #define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__) #endif #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ typedef struct { ethr_mutex mtx; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_ref_t lcnt; #endif #ifdef DEBUG erts_lock_flags_t flags; #endif } erts_mtx_t; typedef ethr_cond erts_cnd_t; /* rwmutex */ typedef struct { ethr_rwmutex rwmtx; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_ref_t lcnt; #endif #ifdef DEBUG erts_lock_flags_t flags; #endif } erts_rwmtx_t; #define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER #define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER #define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER #define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL #define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ #define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ #define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED #define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED #define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED typedef ethr_rwmutex_opt erts_rwmtx_opt_t; typedef ethr_tsd_key erts_tsd_key_t; typedef ethr_ts_event erts_tse_t; #define erts_dw_aint_t ethr_dw_sint_t #define erts_dw_atomic_t ethr_dw_atomic_t #define erts_aint_t ethr_sint_t #define erts_atomic_t ethr_atomic_t #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t #if defined(ARCH_32) # define erts_atomic64_t ethr_dw_atomic_t # define erts_aint64_t ethr_sint64_t #elif defined(ARCH_64) # define erts_atomic64_t ethr_atomic_t # define erts_aint64_t ethr_sint_t #else # error "Not supported architecture" #endif #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD /* spinlock */ typedef struct { ethr_spinlock_t slck; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_ref_t lcnt; #endif #ifdef DEBUG erts_lock_flags_t flags; #endif } erts_spinlock_t; /* rwlock */ typedef struct { ethr_rwlock_t rwlck; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_ref_t lcnt; #endif #ifdef DEBUG erts_lock_flags_t flags; #endif } erts_rwlock_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER #define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif #define ERTS_AINT_NULL ((erts_aint_t) NULL) #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) #define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) #define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid); ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); #else ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx); #endif ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); #else ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); #endif ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock); ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock); #endif ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock); ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line); ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock); #endif ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo); ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo); ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); ERTS_GLB_INLINE int erts_thr_get_main_status(void); ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset); ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_nob(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_ddrb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_rb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_wb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_acqb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_relb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_mb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_nob(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_rb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_wb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_acqb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_relb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_mb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); #if ERTS_GLB_INLINE_INCL_FUNC_DEF #define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \ do { \ Type act = ReadOp((VarP)); \ while (1) { \ Type exp = act; \ Type new = exp & ~(Mask); \ new |= ((Mask) & (Set)); \ act = CmpxchgOp((VarP), new, exp); \ if (act == exp) \ return act; \ } \ } while (0) #endif ERTS_GLB_INLINE void erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val); ERTS_GLB_INLINE void erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val); ERTS_GLB_INLINE void erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val); ERTS_GLB_INLINE erts_aint_t erts_atomic_read_dirty(erts_atomic_t *var); ERTS_GLB_INLINE void erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val); ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_dirty(erts_atomic32_t *var); /* * See "Documentation of atomics and memory barriers" at the top * of this file for info on atomics. */ /* Double word size atomics */ #define erts_dw_atomic_init_nob ethr_dw_atomic_init #define erts_dw_atomic_set_nob ethr_dw_atomic_set #define erts_dw_atomic_read_nob ethr_dw_atomic_read #define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg #define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb #define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb #define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb #define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb #define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb #define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb #define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb #define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb #define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb #define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb #define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb #define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb #define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb #define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb #define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb #define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb #define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb #define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb #define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val) { ethr_sint_t *sint = ethr_dw_atomic_addr(var); sint[0] = val->sint[0]; sint[1] = val->sint[1]; } ERTS_GLB_INLINE void erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val) { ethr_sint_t *sint = ethr_dw_atomic_addr(var); val->sint[0] = sint[0]; val->sint[1] = sint[1]; } #endif /* Word size atomics */ #define erts_atomic_init_nob ethr_atomic_init #define erts_atomic_set_nob ethr_atomic_set #define erts_atomic_read_nob ethr_atomic_read #define erts_atomic_inc_read_nob ethr_atomic_inc_read #define erts_atomic_dec_read_nob ethr_atomic_dec_read #define erts_atomic_inc_nob ethr_atomic_inc #define erts_atomic_dec_nob ethr_atomic_dec #define erts_atomic_add_read_nob ethr_atomic_add_read #define erts_atomic_add_nob ethr_atomic_add #define erts_atomic_read_bor_nob ethr_atomic_read_bor #define erts_atomic_read_band_nob ethr_atomic_read_band #define erts_atomic_xchg_nob ethr_atomic_xchg #define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_nob(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg, var, mask, set); } #endif #define erts_atomic_init_mb ethr_atomic_init_mb #define erts_atomic_set_mb ethr_atomic_set_mb #define erts_atomic_read_mb ethr_atomic_read_mb #define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb #define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb #define erts_atomic_inc_mb ethr_atomic_inc_mb #define erts_atomic_dec_mb ethr_atomic_dec_mb #define erts_atomic_add_read_mb ethr_atomic_add_read_mb #define erts_atomic_add_mb ethr_atomic_add_mb #define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb #define erts_atomic_read_band_mb ethr_atomic_read_band_mb #define erts_atomic_xchg_mb ethr_atomic_xchg_mb #define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_mb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_mb, var, mask, set); } #endif #define erts_atomic_init_acqb ethr_atomic_init_acqb #define erts_atomic_set_acqb ethr_atomic_set_acqb #define erts_atomic_read_acqb ethr_atomic_read_acqb #define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb #define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb #define erts_atomic_inc_acqb ethr_atomic_inc_acqb #define erts_atomic_dec_acqb ethr_atomic_dec_acqb #define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb #define erts_atomic_add_acqb ethr_atomic_add_acqb #define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb #define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb #define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb #define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_acqb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_acqb, var, mask, set); } #endif #define erts_atomic_init_relb ethr_atomic_init_relb #define erts_atomic_set_relb ethr_atomic_set_relb #define erts_atomic_read_relb ethr_atomic_read_relb #define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb #define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb #define erts_atomic_inc_relb ethr_atomic_inc_relb #define erts_atomic_dec_relb ethr_atomic_dec_relb #define erts_atomic_add_read_relb ethr_atomic_add_read_relb #define erts_atomic_add_relb ethr_atomic_add_relb #define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb #define erts_atomic_read_band_relb ethr_atomic_read_band_relb #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_relb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_relb, var, mask, set); } #endif #define erts_atomic_init_ddrb ethr_atomic_init_ddrb #define erts_atomic_set_ddrb ethr_atomic_set_ddrb #define erts_atomic_read_ddrb ethr_atomic_read_ddrb #define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb #define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb #define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb #define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb #define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb #define erts_atomic_add_ddrb ethr_atomic_add_ddrb #define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb #define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb #define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb #define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_ddrb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_ddrb, var, mask, set); } #endif #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb #define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb #define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb #define erts_atomic_inc_rb ethr_atomic_inc_rb #define erts_atomic_dec_rb ethr_atomic_dec_rb #define erts_atomic_add_read_rb ethr_atomic_add_read_rb #define erts_atomic_add_rb ethr_atomic_add_rb #define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb #define erts_atomic_read_band_rb ethr_atomic_read_band_rb #define erts_atomic_xchg_rb ethr_atomic_xchg_rb #define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_rb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_rb, var, mask, set); } #endif #define erts_atomic_init_wb ethr_atomic_init_wb #define erts_atomic_set_wb ethr_atomic_set_wb #define erts_atomic_read_wb ethr_atomic_read_wb #define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb #define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb #define erts_atomic_inc_wb ethr_atomic_inc_wb #define erts_atomic_dec_wb ethr_atomic_dec_wb #define erts_atomic_add_read_wb ethr_atomic_add_read_wb #define erts_atomic_add_wb ethr_atomic_add_wb #define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb #define erts_atomic_read_band_wb ethr_atomic_read_band_wb #define erts_atomic_xchg_wb ethr_atomic_xchg_wb #define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_wb(erts_atomic_t *var, erts_aint_t mask, erts_aint_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint_t, ethr_atomic_read, ethr_atomic_cmpxchg_wb, var, mask, set); } #endif #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val) { ethr_sint_t *sint = ethr_atomic_addr(var); *sint = val; } ERTS_GLB_INLINE erts_aint_t erts_atomic_read_dirty(erts_atomic_t *var) { ethr_sint_t *sint = ethr_atomic_addr(var); return *sint; } #endif /* 32-bit atomics */ #define erts_atomic32_init_nob ethr_atomic32_init #define erts_atomic32_set_nob ethr_atomic32_set #define erts_atomic32_read_nob ethr_atomic32_read #define erts_atomic32_inc_read_nob ethr_atomic32_inc_read #define erts_atomic32_dec_read_nob ethr_atomic32_dec_read #define erts_atomic32_inc_nob ethr_atomic32_inc #define erts_atomic32_dec_nob ethr_atomic32_dec #define erts_atomic32_add_read_nob ethr_atomic32_add_read #define erts_atomic32_add_nob ethr_atomic32_add #define erts_atomic32_read_bor_nob ethr_atomic32_read_bor #define erts_atomic32_read_band_nob ethr_atomic32_read_band #define erts_atomic32_xchg_nob ethr_atomic32_xchg #define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_nob(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg, var, mask, set); } #endif #define erts_atomic32_init_mb ethr_atomic32_init_mb #define erts_atomic32_set_mb ethr_atomic32_set_mb #define erts_atomic32_read_mb ethr_atomic32_read_mb #define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb #define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb #define erts_atomic32_inc_mb ethr_atomic32_inc_mb #define erts_atomic32_dec_mb ethr_atomic32_dec_mb #define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb #define erts_atomic32_add_mb ethr_atomic32_add_mb #define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb #define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb #define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb #define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_mb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_mb, var, mask, set); } #endif #define erts_atomic32_init_acqb ethr_atomic32_init_acqb #define erts_atomic32_set_acqb ethr_atomic32_set_acqb #define erts_atomic32_read_acqb ethr_atomic32_read_acqb #define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb #define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb #define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb #define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb #define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb #define erts_atomic32_add_acqb ethr_atomic32_add_acqb #define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb #define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb #define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb #define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_acqb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_acqb, var, mask, set); } #endif #define erts_atomic32_init_relb ethr_atomic32_init_relb #define erts_atomic32_set_relb ethr_atomic32_set_relb #define erts_atomic32_read_relb ethr_atomic32_read_relb #define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb #define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb #define erts_atomic32_inc_relb ethr_atomic32_inc_relb #define erts_atomic32_dec_relb ethr_atomic32_dec_relb #define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb #define erts_atomic32_add_relb ethr_atomic32_add_relb #define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb #define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_relb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_relb, var, mask, set); } #endif #define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb #define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb #define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb #define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb #define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb #define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb #define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb #define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb #define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb #define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb #define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb #define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb #define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_ddrb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_ddrb, var, mask, set); } #endif #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb #define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb #define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb #define erts_atomic32_inc_rb ethr_atomic32_inc_rb #define erts_atomic32_dec_rb ethr_atomic32_dec_rb #define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb #define erts_atomic32_add_rb ethr_atomic32_add_rb #define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb #define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb #define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb #define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_rb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_rb, var, mask, set); } #endif #define erts_atomic32_init_wb ethr_atomic32_init_wb #define erts_atomic32_set_wb ethr_atomic32_set_wb #define erts_atomic32_read_wb ethr_atomic32_read_wb #define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb #define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb #define erts_atomic32_inc_wb ethr_atomic32_inc_wb #define erts_atomic32_dec_wb ethr_atomic32_dec_wb #define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb #define erts_atomic32_add_wb ethr_atomic32_add_wb #define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb #define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb #define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb #define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_bset_wb(erts_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set) { ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t, ethr_atomic32_read, ethr_atomic32_cmpxchg_wb, var, mask, set); } #endif #undef ERTS_ATOMIC_BSET_IMPL__ #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val) { ethr_sint32_t *sint = ethr_atomic32_addr(var); *sint = val; } ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_dirty(erts_atomic32_t *var) { ethr_sint32_t *sint = ethr_atomic32_addr(var); return *sint; } #endif /* 64-bit atomics */ #if defined(ARCH_64) #define erts_atomic64_init_nob ethr_atomic_init #define erts_atomic64_set_nob ethr_atomic_set #define erts_atomic64_read_nob ethr_atomic_read #define erts_atomic64_inc_read_nob ethr_atomic_inc_read #define erts_atomic64_dec_read_nob ethr_atomic_dec_read #define erts_atomic64_inc_nob ethr_atomic_inc #define erts_atomic64_dec_nob ethr_atomic_dec #define erts_atomic64_add_read_nob ethr_atomic_add_read #define erts_atomic64_add_nob ethr_atomic_add #define erts_atomic64_read_bor_nob ethr_atomic_read_bor #define erts_atomic64_read_band_nob ethr_atomic_read_band #define erts_atomic64_xchg_nob ethr_atomic_xchg #define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg #define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob #define erts_atomic64_init_mb ethr_atomic_init_mb #define erts_atomic64_set_mb ethr_atomic_set_mb #define erts_atomic64_read_mb ethr_atomic_read_mb #define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb #define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb #define erts_atomic64_inc_mb ethr_atomic_inc_mb #define erts_atomic64_dec_mb ethr_atomic_dec_mb #define erts_atomic64_add_read_mb ethr_atomic_add_read_mb #define erts_atomic64_add_mb ethr_atomic_add_mb #define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb #define erts_atomic64_read_band_mb ethr_atomic_read_band_mb #define erts_atomic64_xchg_mb ethr_atomic_xchg_mb #define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb #define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb #define erts_atomic64_init_acqb ethr_atomic_init_acqb #define erts_atomic64_set_acqb ethr_atomic_set_acqb #define erts_atomic64_read_acqb ethr_atomic_read_acqb #define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb #define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb #define erts_atomic64_inc_acqb ethr_atomic_inc_acqb #define erts_atomic64_dec_acqb ethr_atomic_dec_acqb #define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb #define erts_atomic64_add_acqb ethr_atomic_add_acqb #define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb #define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb #define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb #define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb #define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb #define erts_atomic64_init_relb ethr_atomic_init_relb #define erts_atomic64_set_relb ethr_atomic_set_relb #define erts_atomic64_read_relb ethr_atomic_read_relb #define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb #define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb #define erts_atomic64_inc_relb ethr_atomic_inc_relb #define erts_atomic64_dec_relb ethr_atomic_dec_relb #define erts_atomic64_add_read_relb ethr_atomic_add_read_relb #define erts_atomic64_add_relb ethr_atomic_add_relb #define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb #define erts_atomic64_read_band_relb ethr_atomic_read_band_relb #define erts_atomic64_xchg_relb ethr_atomic_xchg_relb #define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb #define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb #define erts_atomic64_init_ddrb ethr_atomic_init_ddrb #define erts_atomic64_set_ddrb ethr_atomic_set_ddrb #define erts_atomic64_read_ddrb ethr_atomic_read_ddrb #define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb #define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb #define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb #define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb #define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb #define erts_atomic64_add_ddrb ethr_atomic_add_ddrb #define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb #define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb #define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb #define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb #define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb #define erts_atomic64_init_rb ethr_atomic_init_rb #define erts_atomic64_set_rb ethr_atomic_set_rb #define erts_atomic64_read_rb ethr_atomic_read_rb #define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb #define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb #define erts_atomic64_inc_rb ethr_atomic_inc_rb #define erts_atomic64_dec_rb ethr_atomic_dec_rb #define erts_atomic64_add_read_rb ethr_atomic_add_read_rb #define erts_atomic64_add_rb ethr_atomic_add_rb #define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb #define erts_atomic64_read_band_rb ethr_atomic_read_band_rb #define erts_atomic64_xchg_rb ethr_atomic_xchg_rb #define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb #define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb #define erts_atomic64_init_wb ethr_atomic_init_wb #define erts_atomic64_set_wb ethr_atomic_set_wb #define erts_atomic64_read_wb ethr_atomic_read_wb #define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb #define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb #define erts_atomic64_inc_wb ethr_atomic_inc_wb #define erts_atomic64_dec_wb ethr_atomic_dec_wb #define erts_atomic64_add_read_wb ethr_atomic_add_read_wb #define erts_atomic64_add_wb ethr_atomic_add_wb #define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb #define erts_atomic64_read_band_wb ethr_atomic_read_band_wb #define erts_atomic64_xchg_wb ethr_atomic_xchg_wb #define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb #define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb #define erts_atomic64_set_dirty erts_atomic_set_dirty #define erts_atomic64_read_dirty erts_atomic_read_dirty #elif defined(ARCH_32) #undef ERTS_ATOMIC64_OPS_DECL__ #define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \ ERTS_GLB_INLINE void \ erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE void \ erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \ ERTS_GLB_INLINE void \ erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \ ERTS_GLB_INLINE void \ erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE void \ erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t new, \ erts_aint64_t exp); \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t mask, \ erts_aint64_t set) ERTS_ATOMIC64_OPS_DECL__(nob); ERTS_ATOMIC64_OPS_DECL__(mb); ERTS_ATOMIC64_OPS_DECL__(acqb); ERTS_ATOMIC64_OPS_DECL__(relb); ERTS_ATOMIC64_OPS_DECL__(ddrb); ERTS_ATOMIC64_OPS_DECL__(rb); ERTS_ATOMIC64_OPS_DECL__(wb); #undef ERTS_ATOMIC64_OPS_DECL__ ERTS_GLB_INLINE void erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val); ERTS_GLB_INLINE erts_aint64_t erts_atomic64_read_dirty(erts_atomic64_t *var); #if ERTS_GLB_INLINE_INCL_FUNC_DEF /* * The ethr_dw_atomic_*_nob() functions below * are here to make it possible for the * ERTS_ATOMIC64_OPS_IMPL__() to map erts * barriers to ethread barriers... */ static ERTS_INLINE void ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ethr_dw_atomic_init(var, val); } static ERTS_INLINE void ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ethr_dw_atomic_set(var, val); } static ERTS_INLINE void ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ethr_dw_atomic_read(var, val); } static ERTS_INLINE int ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var, ethr_dw_sint_t *new, ethr_dw_sint_t *xchg) { return ethr_dw_atomic_cmpxchg(var, new, xchg); } #undef ERTS_ATOMIC64_OPS_IMPL__ #undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ #undef ERTS_DW_SINT_TO_AINT64__ #undef ERTS_AINT64_TO_DW_SINT__ #ifdef ETHR_SU_DW_NAINT_T__ #define ERTS_DW_SINT_TO_AINT64__(DW) \ ((erts_aint64_t) DW.dw_sint) #define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64) #else /* !ETHR_SU_DW_NAINT_T__ */ #define ERTS_DW_SINT_TO_AINT64__(DW) \ ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \ | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \ & ((erts_aint64_t) 0xffffffff))) #define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ do { \ DW.sint[ETHR_DW_SINT_LOW_WORD] = \ (ethr_sint_t) (AINT64 & 0xffffffff); \ DW.sint[ETHR_DW_SINT_HIGH_WORD] = \ (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \ } while (0) #endif /* !ETHR_SU_DW_NAINT_T__ */ #define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \ AVarP, XchgVar, NewVar, \ ModificationCode) \ do { \ ethr_dw_sint_t dw_xchg__, dw_new__; \ ethr_dw_atomic_read(AVarP, &dw_xchg__); \ do { \ XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \ { \ ModificationCode; \ } \ ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \ } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \ } while (0) #define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \ \ ERTS_GLB_INLINE void \ erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ ethr_dw_sint_t dw; \ ERTS_AINT64_TO_DW_SINT__(dw, val); \ ethr_dw_atomic_init_ ## BARRIER(var, &dw); \ } \ \ ERTS_GLB_INLINE void \ erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ ethr_dw_sint_t dw; \ ERTS_AINT64_TO_DW_SINT__(dw, val); \ ethr_dw_atomic_set_ ## BARRIER(var, &dw); \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \ { \ ethr_dw_sint_t dw; \ ethr_dw_atomic_read_ ## BARRIER(var, &dw); \ return ERTS_DW_SINT_TO_AINT64__(dw); \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg + 1); \ return new; \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg - 1); \ return new; \ } \ \ ERTS_GLB_INLINE void \ erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg + 1); \ } \ \ ERTS_GLB_INLINE void \ erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg - 1); \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg + val); \ return new; \ } \ \ ERTS_GLB_INLINE void \ erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg + val); \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg | val); \ return xchg; \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = xchg & val); \ return xchg; \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t val) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ new = val); \ return xchg; \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t new, \ erts_aint64_t exp) \ { \ ethr_dw_sint_t dw_xchg, dw_new; \ ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \ ERTS_AINT64_TO_DW_SINT__(dw_new, new); \ if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \ return exp; \ return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \ } \ \ ERTS_GLB_INLINE erts_aint64_t \ erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ erts_aint64_t mask, \ erts_aint64_t set) \ { \ erts_aint64_t xchg, new; \ ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ var, xchg, new, \ { \ new = xchg & ~mask; \ new |= mask & set; \ }); \ return xchg; \ } ERTS_ATOMIC64_OPS_IMPL__(nob) ERTS_ATOMIC64_OPS_IMPL__(mb) ERTS_ATOMIC64_OPS_IMPL__(acqb) ERTS_ATOMIC64_OPS_IMPL__(relb) ERTS_ATOMIC64_OPS_IMPL__(ddrb) ERTS_ATOMIC64_OPS_IMPL__(rb) ERTS_ATOMIC64_OPS_IMPL__(wb) #undef ERTS_ATOMIC64_OPS_IMPL__ #undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ ERTS_GLB_INLINE void erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val) { ethr_sint_t *sint = ethr_dw_atomic_addr(var); ethr_dw_sint_t dw; ERTS_AINT64_TO_DW_SINT__(dw, val); sint[0] = dw.sint[0]; sint[1] = dw.sint[1]; } ERTS_GLB_INLINE erts_aint64_t erts_atomic64_read_dirty(erts_atomic64_t *var) { ethr_sint_t *sint; ethr_dw_sint_t dw; sint = ethr_dw_atomic_addr(var); dw.sint[0] = sint[0]; dw.sint[1] = sint[1]; return ERTS_DW_SINT_TO_AINT64__(dw); } #undef ERTS_DW_SINT_TO_AINT64__ #undef ERTS_AINT64_TO_DW_SINT__ #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ARCH_32 */ #include "erl_msacc.h" #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id) { int res = ethr_init(id); if (res) erts_thr_fatal_error(res, "initialize thread library"); } ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id) { int res = ethr_late_init(id); if (res) erts_thr_fatal_error(res, "complete initialization of thread library"); } ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { int res = ethr_thr_create(tid, func, arg, opts); if (res) erts_thr_fatal_error(res, "create thread"); } ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res) { int res = ethr_thr_join(tid, thr_res); if (res) erts_thr_fatal_error(res, "join thread"); } ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid) { int res = ethr_thr_detach(tid); if (res) erts_thr_fatal_error(res, "detach thread"); } ERTS_GLB_INLINE void erts_thr_exit(void *res) { ethr_thr_exit(res); erts_thr_fatal_error(0, "terminate thread"); } ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)) { int res = ethr_install_exit_handler(exit_handler); if (res != 0) erts_thr_fatal_error(res, "install thread exit handler"); } ERTS_GLB_INLINE erts_tid_t erts_thr_self(void) { return ethr_self(); } ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len) { return ethr_getname(tid, buf, len); } ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y) { return ethr_equal_tids(x, y); } ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags) { int res = ethr_mutex_init(&mtx->mtx); if (res) { erts_thr_fatal_error(res, "initialize mutex"); } flags |= ERTS_LOCK_TYPE_MUTEX; #ifdef DEBUG mtx->flags = flags; #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init_lock_x(&mtx->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags) { erts_mtx_init(mtx, name, extra, flags); ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, 1); #endif } ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx) { int res; ASSERT(!(mtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_uninstall(&mtx->lcnt); #endif res = ethr_mutex_destroy(&mtx->mtx); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG if (res == EBUSY) { char *warn = "Ignoring busy mutex destroy. " "Most likely a bug in pthread implementation."; erts_send_warning_to_logger_str_nogl(warn); } else #endif erts_thr_fatal_error(res, "destroy mutex"); } } ERTS_GLB_INLINE int #ifdef ERTS_ENABLE_LOCK_POSITION erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line) #else erts_mtx_trylock(erts_mtx_t *mtx) #endif { int res; #ifdef ERTS_ENABLE_LOCK_CHECK if (erts_lc_trylock_force_busy(&mtx->lc)) return EBUSY; /* Make sure caller can handle the situation without causing a lock order violation */ #endif res = ethr_mutex_trylock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_trylock_x(res == 0, &mtx->lc,file,line); #else erts_lc_trylock(res == 0, &mtx->lc); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); #endif return res; } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line) #else erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_x(&mtx->lc, file, line); #else erts_lc_lock(&mtx->lc); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif } ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif ethr_mutex_unlock(&mtx->mtx); } ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; lc.flags = ERTS_LOCK_FLAGS_TYPE_MUTEX; lc.taken_options = 0; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd) { int res = ethr_cond_init(cnd); if (res) erts_thr_fatal_error(res, "initialize condition variable"); } ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd) { int res = ethr_cond_destroy(cnd); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG if (res == EBUSY) { char *warn = "Ignoring busy cond destroy. " "Most likely a bug in pthread implementation."; erts_send_warning_to_logger_str_nogl(warn); } else #endif erts_thr_fatal_error(res, "destroy condition variable"); } } ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) { int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif res = ethr_cond_wait(cnd, &mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post(&mtx->lcnt); #endif if (res != 0 && res != EINTR) erts_thr_fatal_error(res, "wait on condition variable"); ERTS_MSACC_POP_STATE(); } /* * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast() * * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' * even though the associated mutex/mutexes isn't/aren't locked by the * caller. Our implementation do not allow that in order to avoid a * performance penalty. That is, all associated mutexes *need* to be * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()! */ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { ethr_cond_signal(cnd); } ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { ethr_cond_broadcast(cnd); } /* rwmutex */ ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no) { int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_no_locked_of_type(ERTS_LOCK_TYPE_RWMUTEX); #endif res = ethr_rwmutex_set_reader_group(no); if (res != 0) erts_thr_fatal_error(res, "set reader group"); } ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt, char *name, Eterm extra, erts_lock_flags_t flags) { int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) { erts_thr_fatal_error(res, "initialize rwmutex"); } flags |= ERTS_LOCK_TYPE_RWMUTEX; #ifdef DEBUG rwmtx->flags = flags; #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init_lock_x(&rwmtx->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra, erts_lock_flags_t flags) { erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags); } ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { int res; ASSERT(!(rwmtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&rwmtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_uninstall(&rwmtx->lcnt); #endif res = ethr_rwmutex_destroy(&rwmtx->rwmtx); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG if (res == EBUSY) { char *warn = "Ignoring busy rwmutex destroy. " "Most likely a bug in pthread implementation."; erts_send_warning_to_logger_str_nogl(warn); } else #endif erts_thr_fatal_error(res, "destroy rwmutex"); } } ERTS_GLB_INLINE int #ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #endif { int res; #ifdef ERTS_ENABLE_LOCK_CHECK if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ)) return EBUSY; /* Make sure caller can handle the situation without causing a lock order violation */ #endif res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_READ); #endif return res; } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif } ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_runlock(&rwmtx->rwmtx); } ERTS_GLB_INLINE int #ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #endif { int res; #ifdef ERTS_ENABLE_LOCK_CHECK if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR)) return EBUSY; /* Make sure caller can handle the situation without causing a lock order violation */ #endif res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_RDWR); #endif return res; } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) #else erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif } ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwunlock(&rwmtx->rwmtx); } #if 0 /* The following rwmtx function names are reserved for potential future use. */ /* Try upgrade from r-locked state to rw-locked state */ ERTS_GLB_INLINE int erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx) { return 0; } /* Upgrade from r-locked state to rw-locked state */ ERTS_GLB_INLINE void erts_rwmtx_wlock(erts_rwmtx_t *rwmtx) { } /* Downgrade from rw-locked state to r-locked state */ ERTS_GLB_INLINE void erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx) { } #endif ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; lc.flags = ERTS_LOCK_TYPE_RWMUTEX; lc.taken_options = ERTS_LOCK_OPTIONS_READ; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; lc.flags = ERTS_LOCK_TYPE_RWMUTEX; lc.taken_options = ERTS_LOCK_OPTIONS_RDWR; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } /* spinlock */ ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags) { int res = ethr_spinlock_init(&lock->slck); if (res) { erts_thr_fatal_error(res, "init spinlock"); } flags |= ERTS_LOCK_TYPE_SPINLOCK; #ifdef DEBUG lock->flags = flags; #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init_lock_x(&lock->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock) { int res; ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_spinlock_destroy(&lock->slck); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG if (res == EBUSY) { char *warn = "Ignoring busy spinlock destroy. " "Most likely a bug in pthread implementation."; erts_send_warning_to_logger_str_nogl(warn); } else #endif erts_thr_fatal_error(res, "destroy rwlock"); } } ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif ethr_spin_unlock(&lock->slck); } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line) #else erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_x(&lock->lc,file,line); #else erts_lc_lock(&lock->lc); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif } ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; lc.flags = ERTS_LOCK_TYPE_SPINLOCK; lc.taken_options = 0; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } /* rwspinlock */ ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags) { int res = ethr_rwlock_init(&lock->rwlck); if (res) { erts_thr_fatal_error(res, "init rwlock"); } flags |= ERTS_LOCK_TYPE_RWSPINLOCK; #ifdef DEBUG lock->flags = flags; #endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_init_lock_x(&lock->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock) { int res; ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_rwlock_destroy(&lock->rwlck); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG if (res == EBUSY) { char *warn = "Ignoring busy rwlock destroy. " "Most likely a bug in pthread implementation."; erts_send_warning_to_logger_str_nogl(warn); } else #endif erts_thr_fatal_error(res, "destroy rwlock"); } } ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_read_unlock(&lock->rwlck); } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) #else erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif } ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_write_unlock(&lock->rwlck); } ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) #else erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif } ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK; lc.taken_options = ERTS_LOCK_OPTIONS_READ; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) { #if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK; lc.taken_options = ERTS_LOCK_OPTIONS_RDWR; erts_lc_have_locks(&res, &lc, 1); return res; #else return 0; #endif } ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname) { int res = ethr_tsd_key_create(keyp, keyname); if (res) erts_thr_fatal_error(res, "create thread specific data key"); } ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key) { int res = ethr_tsd_key_delete(key); if (res) erts_thr_fatal_error(res, "delete thread specific data key"); } ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value) { int res = ethr_tsd_set(key, value); if (res) erts_thr_fatal_error(res, "set thread specific data"); } ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key) { return ethr_tsd_get(key); } ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { return (erts_tse_t *) ethr_get_ts_event(); } ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { ethr_leave_ts_event(ep); } ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep) { int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event); if (res != 0) erts_thr_fatal_error(res, "prepare timed"); } ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { ethr_event_set(&((ethr_ts_event *) ep)->event); } ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { ethr_event_reset(&((ethr_ts_event *) ep)->event); } ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) { int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_wait(&((ethr_ts_event *) ep)->event); ERTS_MSACC_POP_STATE(); return res; } ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) { int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); ERTS_MSACC_POP_STATE(); return res; } ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) { int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_twait(&((ethr_ts_event *) ep)->event, (ethr_sint64_t) tmo); ERTS_MSACC_POP_STATE(); return res; } ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) { int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_stwait(&((ethr_ts_event *) ep)->event, spincount, (ethr_sint64_t) tmo); ERTS_MSACC_POP_STATE(); return res; } ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; } ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) { int res = ethr_set_main_thr_status(on, no); if (res != 0) erts_thr_fatal_error(res, "set thread main status"); } ERTS_GLB_INLINE int erts_thr_get_main_status(void) { int main_status; int res = ethr_get_main_thr_status(&main_status); if (res != 0) erts_thr_fatal_error(res, "get thread main status"); return main_status; } ERTS_GLB_INLINE void erts_thr_yield(void) { int res = ETHR_YIELD(); if (res != 0) erts_thr_fatal_error(res, "yield"); } #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig) { int res = ethr_kill((ethr_tid)tid, sig); if (res) erts_thr_fatal_error(res, "killing thread"); } ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset) { int res = ethr_sigmask(how, set, oset); if (res) erts_thr_fatal_error(res, "get or set signal mask"); } ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig) { int res; do { res = ethr_sigwait(set, sig); } while (res == EINTR); if (res) erts_thr_fatal_error(res, "to wait for signal"); } #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifndef ERL_THREAD_H__ */