diff options
Diffstat (limited to 'erts')
71 files changed, 11248 insertions, 6841 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 61244c7cd3..0fa3fec244 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -653,6 +653,19 @@ fi ]) +AC_DEFUN(ERL_INTERNAL_LIBS, +[ + +ERTS_INTERNAL_X_LIBS= + +AC_CHECK_LIB(kstat, kstat_open, +[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) +ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) + +AC_SUBST(ERTS_INTERNAL_X_LIBS) + +]) + dnl ---------------------------------------------------------------------- dnl dnl ERL_FIND_ETHR_LIB @@ -676,10 +689,13 @@ AC_DEFUN(ERL_FIND_ETHR_LIB, [ LM_CHECK_THR_LIB +ERL_INTERNAL_LIBS +ethr_have_native_atomics=no +ethr_have_native_spinlock=no ETHR_THR_LIB_BASE="$THR_LIB_NAME" ETHR_DEFS="$THR_DEFS" -ETHR_X_LIBS="$THR_LIBS" +ETHR_X_LIBS="$THR_LIBS $ERTS_INTERNAL_X_LIBS" ETHR_LIBS= ETHR_LIB_NAME= @@ -691,6 +707,7 @@ ethr_lib_name=ethread case "$THR_LIB_NAME" in win32_threads) + ETHR_THR_LIB_BASE_DIR=win # * _WIN32_WINNT >= 0x0400 is needed for # TryEnterCriticalSection # * _WIN32_WINNT >= 0x0403 is needed for @@ -716,10 +733,13 @@ case "$THR_LIB_NAME" in if test $found_win32_winnt = no; then AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS]) fi + ethr_have_native_atomics=yes + ethr_have_native_spinlock=yes AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) ;; pthread) + ETHR_THR_LIB_BASE_DIR=pthread AC_DEFINE(ETHR_PTHREADS, 1, [Define if you have pthreads]) case $host_os in openbsd*) @@ -775,9 +795,7 @@ case "$THR_LIB_NAME" in if test $usable_sigaltstack = no; then ETHR_DEFS="$ETHR_DEFS -DETHR_UNUSABLE_SIGALTSTACK" fi - - AC_DEFINE(ETHR_INIT_MUTEX_IN_CHILD_AT_FORK, 1, \ -[Define if mutexes should be reinitialized (instead of unlocked) in child at fork.]) ;; + ;; *) ;; esac @@ -808,6 +826,10 @@ case "$THR_LIB_NAME" in [Define if you need the <nptl/pthread.h> header file.]) fi + AC_CHECK_HEADER(sched.h, \ + AC_DEFINE(ETHR_HAVE_SCHED_H, 1, \ +[Define if you have the <sched.h> header file.])) + AC_CHECK_HEADER(sys/time.h, \ AC_DEFINE(ETHR_HAVE_SYS_TIME_H, 1, \ [Define if you have the <sys/time.h> header file.])) @@ -823,26 +845,63 @@ case "$THR_LIB_NAME" in dnl Check for functions dnl - AC_CHECK_FUNC(pthread_atfork, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_ATFORK, 1, \ -[Define if you have the pthread_atfork function.])) - AC_CHECK_FUNC(pthread_mutexattr_settype, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE, 1, \ -[Define if you have the pthread_mutexattr_settype function.])) - AC_CHECK_FUNC(pthread_mutexattr_setkind_np, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP, 1, \ -[Define if you have the pthread_mutexattr_setkind_np function.])) AC_CHECK_FUNC(pthread_spin_lock, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ -[Define if you have the pthread_spin_lock function.])) + [ethr_have_native_spinlock=yes \ + AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ +[Define if you have the pthread_spin_lock function.])]) + + have_sched_yield=no + have_librt_sched_yield=no + AC_CHECK_FUNC(sched_yield, [have_sched_yield=yes]) + if test $have_sched_yield = no; then + AC_CHECK_LIB(rt, sched_yield, + [have_librt_sched_yield=yes + ETHR_X_LIBS="$ETHR_X_LIBS -lrt"]) + fi + if test $have_sched_yield = yes || test $have_librt_sched_yield = yes; then + AC_DEFINE(ETHR_HAVE_SCHED_YIELD, 1, [Define if you have the sched_yield() function.]) + AC_MSG_CHECKING([whether sched_yield() returns an int]) + sched_yield_ret_int=no + AC_TRY_COMPILE([ + #ifdef ETHR_HAVE_SCHED_H + #include <sched.h> + #endif + ], + [int sched_yield();], + [sched_yield_ret_int=yes]) + AC_MSG_RESULT([$sched_yield_ret_int]) + if test $sched_yield_ret_int = yes; then + AC_DEFINE(ETHR_SCHED_YIELD_RET_INT, 1, [Define if sched_yield() returns an int.]) + fi + fi + + have_pthread_yield=no + AC_CHECK_FUNC(pthread_yield, [have_pthread_yield=yes]) + if test $have_pthread_yield = yes; then + AC_DEFINE(ETHR_HAVE_PTHREAD_YIELD, 1, [Define if you have the pthread_yield() function.]) + AC_MSG_CHECKING([whether pthread_yield() returns an int]) + pthread_yield_ret_int=no + AC_TRY_COMPILE([ + #if defined(ETHR_NEED_NPTL_PTHREAD_H) + #include <nptl/pthread.h> + #elif defined(ETHR_HAVE_MIT_PTHREAD_H) + #include <pthread/mit/pthread.h> + #elif defined(ETHR_HAVE_PTHREAD_H) + #include <pthread.h> + #endif + ], + [int pthread_yield();], + [pthread_yield_ret_int=yes]) + AC_MSG_RESULT([$pthread_yield_ret_int]) + if test $pthread_yield_ret_int = yes; then + AC_DEFINE(ETHR_PTHREAD_YIELD_RET_INT, 1, [Define if pthread_yield() returns an int.]) + fi + fi have_pthread_rwlock_init=no AC_CHECK_FUNC(pthread_rwlock_init, [have_pthread_rwlock_init=yes]) if test $have_pthread_rwlock_init = yes; then - AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_INIT, 1, \ -[Define if you have a pthread_rwlock implementation that can be used.]) - ethr_have_pthread_rwlockattr_setkind_np=no AC_CHECK_FUNC(pthread_rwlockattr_setkind_np, [ethr_have_pthread_rwlockattr_setkind_np=yes]) @@ -876,12 +935,42 @@ case "$THR_LIB_NAME" in fi fi + if test "$force_pthread_rwlocks" = "yes"; then + AC_DEFINE(ETHR_FORCE_PTHREAD_RWLOCK, 1, \ +[Define if you want to force usage of pthread rwlocks]) + + if test $have_pthread_rwlock_init = yes; then + AC_MSG_WARN([Forced usage of pthread rwlocks. Note that this implementation may suffer from starvation issues.]) + else + AC_MSG_ERROR([User forced usage of pthread rwlock, but no such implementation was found]) + fi + fi AC_CHECK_FUNC(pthread_attr_setguardsize, \ AC_DEFINE(ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE, 1, \ [Define if you have the pthread_attr_setguardsize function.])) + linux_futex=no + AC_MSG_CHECKING([for Linux futexes]) + AC_TRY_LINK([ + #include <sys/syscall.h> + #include <unistd.h> + #include <linux/futex.h> + #include <sys/time.h> + ], + [ + int i = 1; + syscall(__NR_futex, (void *) &i, FUTEX_WAKE, 1, + (void*)0,(void*)0, 0); + syscall(__NR_futex, (void *) &i, FUTEX_WAIT, 0, + (void*)0,(void*)0, 0); + return 0; + ], + linux_futex=yes) + AC_MSG_RESULT([$linux_futex]) + test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.]) + AC_MSG_CHECKING([for GCC atomic operations]) ethr_have_gcc_atomic_ops=no AC_TRY_LINK([], @@ -899,10 +988,69 @@ case "$THR_LIB_NAME" in AC_MSG_RESULT([$ethr_have_gcc_atomic_ops]) test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations]) + case "$host_cpu" in + sun4u | sparc64 | sun4v) + ethr_have_native_atomics=yes;; + i86pc | i386 | i486 | i586 | i686 | x86_64 | amd64) + ethr_have_native_atomics=yes;; + macppc | ppc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac + + AC_MSG_CHECKING([for a usable libatomic_ops implementation]) + case "x$with_libatomic_ops" in + xno | xyes | x) + libatomic_ops_include= + ;; + *) + if test -d "${with_libatomic_ops}/include"; then + libatomic_ops_include="-I$with_libatomic_ops/include" + CPPFLAGS="$CPPFLAGS $libatomic_ops_include" + else + AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found]) + fi;; + esac + ethr_have_libatomic_ops=no + AC_TRY_LINK([#include "atomic_ops.h"], + [ + volatile AO_t x; + AO_t y; + int z; + + AO_nop_full(); + AO_store(&x, (AO_t) 0); + z = AO_load(&x); + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); + ], + [ethr_have_native_atomics=yes + ethr_have_libatomic_ops=yes]) + AC_MSG_RESULT([$ethr_have_libatomic_ops]) + if test $ethr_have_libatomic_ops = yes; then + AC_CHECK_SIZEOF(AO_t, , + [ + #include <stdio.h> + #include "atomic_ops.h" + ]) + AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used]) + + AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations]) + if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations]) + fi + ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include" + elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_MSG_ERROR([No usable libatomic_ops implementation found]) + fi + dnl Restore LIBS LIBS=$saved_libs dnl restore CPPFLAGS CPPFLAGS=$saved_cppflags + ;; *) ;; @@ -918,16 +1066,23 @@ fi if test "x$ETHR_THR_LIB_BASE" != "x"; then ETHR_DEFS="-DUSE_THREADS $ETHR_DEFS" - ETHR_LIBS="-l$ethr_lib_name $ETHR_X_LIBS" + ETHR_LIBS="-l$ethr_lib_name -lerts_internal_r $ETHR_X_LIBS" ETHR_LIB_NAME=$ethr_lib_name fi AC_CHECK_SIZEOF(void *) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers]) -if test "X$disable_native_ethr_impls" = "Xyes"; then - AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) -fi +AC_ARG_ENABLE(native-ethr-impls, + AS_HELP_STRING([--disable-native-ethr-impls], + [disable native ethread implementations]), +[ case "$enableval" in + no) disable_native_ethr_impls=yes ;; + *) disable_native_ethr_impls=no ;; + esac ], disable_native_ethr_impls=no) + +test "X$disable_native_ethr_impls" = "Xyes" && + AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], @@ -940,6 +1095,21 @@ AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, test $enable_prefer_gcc_native_ethr_impls = yes && AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) +AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, + AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], + [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), +[ case "$enableval" in + yes) enable_ethread_pre_pentium4_compatibility=yes ;; + *) enable_ethread_pre_pentium4_compatibilit=no ;; + esac ], enable_ethread_pre_pentium4_compatibilit=no) + +test $enable_ethread_pre_pentium4_compatibilit = yes && + AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.]) + +AC_ARG_WITH(libatomic_ops, + AS_HELP_STRING([--with-libatomic_ops=PATH], + [use libatomic_ops with the ethread library])) + AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) @@ -948,6 +1118,7 @@ AC_SUBST(ETHR_LIBS) AC_SUBST(ETHR_LIB_NAME) AC_SUBST(ETHR_DEFS) AC_SUBST(ETHR_THR_LIB_BASE) +AC_SUBST(ETHR_THR_LIB_BASE_DIR) ]) diff --git a/erts/configure.in b/erts/configure.in index 63bf548c89..157163fe0b 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -61,6 +61,9 @@ if test x"${ERL_TOP}/erts" != x"$srcdir"; then fi erl_top=${ERL_TOP} +# Remove old configuration information +/bin/rm -f "$ERL_TOP/erts/CONF_INFO" + # echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX # echo X # echo "X srcdir = $srcdir" @@ -277,20 +280,6 @@ AC_ARG_ENABLE(clock-gettime, *) clock_gettime_correction=yes ;; esac ], clock_gettime_correction=unknown) -AC_ARG_ENABLE(native-ethr-impls, -[ --enable-native-ethr-impls enable native ethread implementations - --disable-native-ethr-impls disable native ethread implementations], -[ case "$enableval" in - no) disable_native_ethr_impls=yes ;; - *) disable_native_ethr_impls=no ;; - esac ], disable_native_ethr_impls=no) - -dnl Defined in libraries/megaco/configure.in but we need it here -dnl also in order to show it to the "top user" - -AC_ARG_ENABLE(megaco_flex_scanner_lineno, -[ --disable-megaco-flex-scanner-lineno disable megaco flex scanner lineno]) - dnl Magic test for clearcase. OTP_RELEASE= if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then @@ -1049,6 +1038,42 @@ if test $ERTS_BUILD_SMP_EMU = yes; then fi AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built]) + + case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in + yes-*) + ;; + + no-yes) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + No native atomic implementation available. + Fallbacks implemented using spinlocks will be + used. Note that the performance of the SMP + runtime system will suffer due to this. + +EOF + ;; + + no-no) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF + + No native atomic implementation, nor no native + spinlock implementation available. Fallbacks + implemented using mutexes will be used. Note + that the performance of the SMP runtime system + will suffer much due to this. + +EOF + ;; + + esac + enable_threads=force fi @@ -1221,13 +1246,7 @@ fi AC_SUBST(EMU_LOCK_CHECKING) -ERTS_INTERNAL_X_LIBS= - -AC_CHECK_LIB(kstat, kstat_open, -[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) -ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) - -AC_SUBST(ERTS_INTERNAL_X_LIBS) +ERL_INTERNAL_LIBS dnl THR_LIBS and THR_DEFS are only used by odbc THR_LIBS=$ETHR_X_LIBS diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index df80142ce1..0e26d62548 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -603,6 +603,24 @@ <item> <p>Force ets memory block to be moved on realloc.</p> </item> + <tag><marker id="+rg"><c><![CDATA[+rg ReaderGroupsLimit]]></c></marker></tag> + <item> + <p>Limits the amount of reader groups used by read/write locks + optimized for read operations in the Erlang runtime system. By + default the reader groups limit equals 8.</p> + <p>When the amount of schedulers is less than or equal to the reader + groups limit, each scheduler has its own reader group. When the + amount of schedulers is larger than the reader groups limit, + schedulers share reader groups. Shared reader groups degrades + read lock and read unlock performance while a large amount of + reader groups degrades write lock performance, so the limit is a + tradeoff between performance for read operations and performance + for write operations. Each reader group currently consumes 64 byte + in each read/write lock. Also note that a runtime system using + shared reader groups benefits from <seealso marker="#+sbt">binding + schedulers to logical processors</seealso>, since the reader groups + are distributed better between schedulers.</p> + </item> <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag> <item> <p>Sets the amount of scheduler threads to create and scheduler diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index eca6121a1e..2b9f70b0f4 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -199,6 +199,14 @@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ +ifeq ($(TARGET),win32) +LIB_PREFIX= +LIB_SUFFIX=.lib +else +LIB_PREFIX=lib +LIB_SUFFIX=.a +endif + OMIT_OMIT_FP=no ifeq (@EMU_LOCK_CHECKING@,yes) @@ -279,15 +287,12 @@ endif ifeq ($(TARGET),win32) LIBS += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre -DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib else -LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a -DEPLIBS += \ - $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a -# rem liberts_internal.a +LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) endif +DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) + ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK PERFCTR_PATH=@PERFCTR_PATH@ @@ -305,7 +310,19 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS)) +ifneq ($(strip $(THR_LIB_NAME)),) +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ + $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) +else +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) +endif + +THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ + $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) + +LIBS += $(THR_LIBS) + +ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r) ifeq ($(findstring vxworks,$(TARGET)),vxworks) ERTS_INTERNAL_LIB=erts_internal @@ -317,7 +334,9 @@ ERTS_INTERNAL_LIB=erts_internal endif endif -LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) +LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) + +endif # erts_internal_r LIBS += @LIBRT@ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index e2a79d6e4f..6b3c106a97 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock; #define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) #define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) #define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) -#define atom_init_lock() erts_smp_rwmtx_init(&atom_table_lock, \ - "atom_tab") + #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif @@ -304,12 +303,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_init(&atom_put_ops, 0); #endif - atom_init_lock(); + erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 28f69b9460..0815cdbc7f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -432,6 +432,7 @@ atom raw atom re atom re_pattern atom re_run_trap +atom read_concurrency atom ready_input atom ready_output atom ready_async diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 16ae643ed9..87503af7d5 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -64,6 +64,14 @@ #ifdef DEBUG static Uint install_debug_functions(void); +#if 0 +#define HARD_DEBUG +#ifdef __GNUC__ +#warning "* * * * * * * * * * * * * *" +#warning "* HARD DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * *" +#endif +#endif #endif extern void elib_ensure_initialized(void); @@ -391,6 +399,10 @@ refuse_af_strategy(struct au_init *init) static void init_thr_ix(int static_ixs); +#ifdef HARD_DEBUG +static void hdbg_init(void); +#endif + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -406,6 +418,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) ERTS_DEFAULT_ALCU_INIT }; +#ifdef HARD_DEBUG + hdbg_init(); +#endif + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); @@ -2862,12 +2878,10 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0a: - if (ethr_mutex_lock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: - if (ethr_mutex_unlock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); @@ -2883,16 +2897,13 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0e: - if (ethr_cond_broadcast((ethr_cond *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); - if (res != 0) - ERTS_ALC_TEST_ABORT; break; } case 0xf10: { @@ -2939,8 +2950,11 @@ unsigned long erts_alc_test(unsigned long op, #undef PRINT_OPS #endif - +#ifdef HARD_DEBUG +#define FENCE_SZ (4*sizeof(UWord)) +#else #define FENCE_SZ (3*sizeof(UWord)) +#endif #if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 @@ -2962,6 +2976,104 @@ unsigned long erts_alc_test(unsigned long op, #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) +#ifdef HARD_DEBUG + +#define ERL_ALC_HDBG_MAX_MBLK 100000 +#define ERTS_ALC_O_CHECK -1 + +typedef struct hdbg_mblk_ hdbg_mblk; +struct hdbg_mblk_ { + hdbg_mblk *next; + hdbg_mblk *prev; + void *p; + Uint s; + ErtsAlcType_t n; +}; + +static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; + +static hdbg_mblk *free_hdbg_mblks; +static hdbg_mblk *used_hdbg_mblks; +static erts_mtx_t hdbg_mblk_mtx; + +static void +hdbg_init(void) +{ + int i; + for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) + hdbg_mblks[i].next = &hdbg_mblks[i+1]; + hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; + free_hdbg_mblks = &hdbg_mblks[0]; + used_hdbg_mblks = NULL; + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); +} + +static void *check_memory_fence(void *ptr, + Uint *size, + ErtsAlcType_t n, + int func); +void erts_hdbg_chk_blks(void); + +void +erts_hdbg_chk_blks(void) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { + Uint sz; + check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); + ASSERT(sz == mblk->s); + } + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +static hdbg_mblk * +hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + mblk = free_hdbg_mblks; + if (!mblk) { + erts_fprintf(stderr, + "Ran out of debug blocks; please increase " + "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", + ERL_ALC_HDBG_MAX_MBLK); + abort(); + } + free_hdbg_mblks = mblk->next; + + mblk->p = p; + mblk->s = s; + mblk->n = n; + + mblk->next = used_hdbg_mblks; + mblk->prev = NULL; + if (used_hdbg_mblks) + used_hdbg_mblks->prev = mblk; + used_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); + return (void *) mblk; +} + +static void +hdbg_free(hdbg_mblk *mblk) +{ + erts_mtx_lock(&hdbg_mblk_mtx); + if (mblk->next) + mblk->next->prev = mblk->prev; + if (mblk->prev) + mblk->prev->next = mblk->next; + else + used_hdbg_mblks = mblk->next; + + mblk->next = free_hdbg_mblks; + free_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); @@ -3007,17 +3119,28 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { UWord *ui_ptr; UWord pattern; +#ifdef HARD_DEBUG + hdbg_mblk **mblkpp; +#endif if (!ptr) return NULL; ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); - + +#ifdef HARD_DEBUG + mblkpp = (hdbg_mblk **) ui_ptr++; +#endif + *(ui_ptr++) = sz; *(ui_ptr++) = pattern; memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); +#ifdef HARD_DEBUG + *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); +#endif + return (void *) ui_ptr; } @@ -3029,6 +3152,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) UWord pre_pattern; UWord post_pattern; UWord *ui_ptr; +#ifdef HARD_DEBUG + hdbg_mblk *mblk; +#endif if (!ptr) return NULL; @@ -3036,6 +3162,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); +#ifdef HARD_DEBUG + mblk = (hdbg_mblk *) *(--ui_ptr); +#endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { @@ -3091,6 +3220,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } +#ifdef HARD_DEBUG + switch (func) { + case ERTS_ALC_O_REALLOC: + case ERTS_ALC_O_FREE: + hdbg_free(mblk); + break; + default: + break; + } +#endif + return (void *) ui_ptr; } @@ -3103,6 +3243,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint dsize; void *res; +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); @@ -3132,13 +3276,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); - + res = set_memory_fence(res, size, n); #ifdef PRINT_OPS @@ -3168,6 +3316,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + } static Uint diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 6f88bbe5b8..7df9f19af0 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -212,7 +212,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED PROCESSES scheduler_data +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle @@ -246,6 +247,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib +type RDR_GRPS_MAP LONG_LIVED SYSTEM reader_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -269,7 +271,9 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +if threads -type ETHR_INTERNAL SYSTEM SYSTEM ethread_internal +type ETHR_STD STANDARD SYSTEM ethread_standard +type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived +type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived +ifnot smp diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 48cda52612..f582f4470f 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2563,6 +2563,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_sched_stat_term(BIF_P, 1)); } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) { BIF_RET(erts_nif_taints(BIF_P)); + } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) { + BIF_RET(erts_get_reader_groups_map(BIF_P)); } BIF_ERROR(BIF_P, BADARG); @@ -3426,6 +3428,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) { return erts_fake_scheduler_bindings(BIF_P, tp[2]); } + else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) { + Sint groups; + if (is_not_small(tp[2])) + BIF_ERROR(BIF_P, BADARG); + groups = signed_val(tp[2]); + if (groups < (Sint) 1 || groups > (Sint) INT_MAX) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); + } break; } default: @@ -3730,8 +3742,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - ethr_atomic_read(&stats->tries, (long *)&tries); - ethr_atomic_read(&stats->colls, (long *)&colls); + tries = (unsigned long) ethr_atomic_read(&stats->tries); + colls = (unsigned long) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index cbdaa459de..b0369a402b 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -78,11 +78,19 @@ enum DbIterSafety { ** The main meta table, containing all ets tables. */ #ifdef ERTS_SMP -# define META_MAIN_TAB_LOCK_CNT 16 -static union { - erts_smp_spinlock_t lck; - byte _cache_line_alignment[64]; -}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT]; + +#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 +#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) +#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) + +typedef union { + erts_smp_rwmtx_t rwmtx; + byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_smp_rwmtx_t))]; +} erts_meta_main_tab_lock_t; + +static erts_meta_main_tab_lock_t *meta_main_tab_locks; + #endif static struct { union { @@ -104,17 +112,13 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ -static ERTS_INLINE void meta_main_tab_lock(unsigned slot) +static ERTS_INLINE erts_smp_rwmtx_t * +get_meta_main_tab_lock(unsigned slot) { #ifdef ERTS_SMP - erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); -#endif -} - -static ERTS_INLINE void meta_main_tab_unlock(unsigned slot) -{ -#ifdef ERTS_SMP - erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); + return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; +#else + return NULL; #endif } @@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + LCK_NONE=4 } db_lock_kind_t; extern DbTableMethod db_hash; @@ -214,17 +219,17 @@ Export ets_select_continue_exp; */ static Export ets_delete_continue_exp; -static ERTS_INLINE DbTable* db_ref(DbTable* tb) +static ERTS_INLINE DbTable* db_ref(DbTable* tb, db_lock_kind_t kind) { - if (tb != NULL) { + if (tb != NULL && kind != LCK_READ) { erts_refc_inc(&tb->common.ref, 2); } return tb; } -static ERTS_INLINE DbTable* db_unref(DbTable* tb) +static ERTS_INLINE DbTable* db_unref(DbTable* tb, db_lock_kind_t kind) { - if (!erts_refc_dectest(&tb->common.ref, 0)) { + if (kind != LCK_READ && !erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", @@ -256,12 +261,19 @@ static ERTS_INLINE DbTable* db_unref(DbTable* tb) return tb; } -static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname) +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, + char *rwname, char* fixname) { +#ifdef ERTS_SMP + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + if (use_frequent_read_lock) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; +#endif erts_refc_init(&tb->common.ref, 1); erts_refc_init(&tb->common.fixref, 0); #ifdef ERTS_SMP - erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name); + erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, + rwname, tb->common.the_name); erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); #endif @@ -297,7 +309,7 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { - (void) db_ref(tb); + (void) db_ref(tb, kind); #ifdef ERTS_SMP db_lock_take_over_ref(tb, kind); #endif @@ -331,7 +343,7 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb); /* May delete table... */ + (void) db_unref(tb, kind); /* May delete table... */ } @@ -349,32 +361,49 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) } static ERTS_INLINE -DbTable* db_get_table(Process *p, - Eterm id, - int what, - db_lock_kind_t kind) +DbTable* db_get_table_aux(Process *p, + Eterm id, + int what, + db_lock_kind_t kind, + int meta_already_locked) { DbTable *tb = NULL; + erts_smp_rwmtx_t *mtl = NULL; if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - meta_main_tab_lock(slot); + if (!meta_already_locked) { + mtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rlock(mtl); + } +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + else { + erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) + || erts_lc_rwmtx_is_rwlocked(test_mtl)); + } +#endif if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { /* SMP: inc to prevent race, between unlock of meta_main_tab_lock * and the table locking outside the meta_main_tab_lock */ - tb = db_ref(meta_main_tab[slot].u.tb); + tb = db_ref(meta_main_tab[slot].u.tb, kind); } - meta_main_tab_unlock(slot); } else if (is_atom(id)) { - erts_smp_rwmtx_t* rwlock; - struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock); - erts_smp_rwmtx_rlock(rwlock); + struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); + if (!meta_already_locked) + erts_smp_rwmtx_rlock(mtl); + else{ + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); + mtl = NULL; + } + if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb); + tb = db_ref(bucket->pu.tb, kind); } } else { /* multi */ @@ -382,23 +411,33 @@ DbTable* db_get_table(Process *p, Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb); + tb = db_ref(bucket->pu.mvec[i].pu.tb, kind); break; } } } } - erts_smp_rwmtx_runlock(rwlock); } if (tb) { db_lock_take_over_ref(tb, kind); - if (tb->common.id == id && ((tb->common.status & what) != 0 || - p->id == tb->common.owner)) { - return tb; + if (tb->common.id != id + || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + db_unlock(tb, kind); + tb = NULL; } - db_unlock(tb, kind); } - return NULL; + if (mtl) + erts_smp_rwmtx_runlock(mtl); + return tb; +} + +static ERTS_INLINE +DbTable* db_get_table(Process *p, + Eterm id, + int what, + db_lock_kind_t kind) +{ + return db_get_table_aux(p, id, what, kind, 0); } /* Requires meta_main_tab_locks[slot] locked. @@ -413,15 +452,15 @@ static ERTS_INLINE void free_slot(int slot) erts_smp_spin_unlock(&meta_main_tab_main_lock); } -static int insert_named_tab(Eterm name_atom, DbTable* tb) +static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - - erts_smp_rwmtx_rwlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -468,17 +507,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } -static int remove_named_tab(Eterm name_atom) +static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; + Eterm name_atom = tb->common.id; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - erts_smp_rwmtx_rwlock(rwlock); +#ifdef ERTS_SMP + if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(rwlock); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + if (bucket->pu.tb == NULL) { goto done; } @@ -529,7 +583,8 @@ static int remove_named_tab(Eterm name_atom) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } @@ -1133,6 +1188,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1141,34 +1197,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { + + if (is_not_atom(BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } - if (is_not_atom(BIF_ARG_2)) { - goto badarg; + (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); + + if (is_small(BIF_ARG_1)) { + Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); + } + else if (is_atom(BIF_ARG_1)) { + (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (lck1 == lck2) + lck2 = NULL; + else if (lck1 > lck2) { + erts_smp_rwmtx_t *tmp = lck1; + lck1 = lck2; + lck2 = tmp; + } + } + else { + BIF_ERROR(BIF_P, BADARG); } + erts_smp_rwmtx_rwlock(lck1); + if (lck2) + erts_smp_rwmtx_rwlock(lck2); + + tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); + if (!tb) + goto badarg; + if (is_not_atom(tb->common.id)) { /* Not a named table */ tb->common.the_name = BIF_ARG_2; goto done; } - if (!insert_named_tab(BIF_ARG_2,tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) goto badarg; - } - if (!remove_named_tab(tb->common.id)) { + + if (!remove_named_tab(tb, 1)) erl_exit(1,"Could not find named tab %s", tb->common.id); - } tb->common.id = tb->common.the_name = BIF_ARG_2; done: ret = tb->common.id; db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: - db_unlock(tb, LCK_WRITE); + if (tb) + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1189,10 +1276,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked; + int is_named, is_fine_locked, frequent_read; int cret; DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; + erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1205,6 +1293,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = 1; is_named = 0; is_fine_locked = 0; + frequent_read = 0; heir = am_none; heir_data = (UWord) am_undefined; @@ -1238,6 +1327,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_fine_locked = 0; } else break; } + else if (tp[1] == am_read_concurrency) { + if (tp[2] == am_true) { + frequent_read = 1; + } else if (tp[2] == am_false) { + frequent_read = 0; + } else break; + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; @@ -1286,6 +1382,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +#ifdef ERTS_SMP + if (frequent_read && !(status & DB_PRIVATE)) + status |= DB_FREQ_READ; +#endif + /* we create table outside any table lock * and take the unusal cost of destroy table if it * fails to find a slot @@ -1308,7 +1409,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - db_init_lock(tb, "db_tab", "db_tab_fix"); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), + "db_tab", "db_tab_fix"); tb->common.keypos = keypos; tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); @@ -1351,15 +1453,17 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ - meta_main_tab_lock(slot); + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; ASSERT(IS_SLOT_ALIVE(slot)); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb)) { - meta_main_tab_lock(slot); + if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); free_slot(slot); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); db_lock_take_over_ref(tb,LCK_WRITE); free_heir_data(tb); @@ -1499,6 +1603,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1520,13 +1625,23 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif /* We must keep the slot, to be found by db_proc_dead() if process dies */ MARK_SLOT_DEAD(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); - if (is_atom(tb->common.id)) { - remove_named_tab(tb->common.id); - } + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); if (tb->common.owner != BIF_P->id) { DeclareTmpHeap(meta_tuple,3,BIF_P); @@ -1919,14 +2034,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) previous = NIL; j = 0; for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { - meta_main_tab_lock(i); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { j++; tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; } - meta_main_tab_unlock(i); + erts_smp_rwmtx_runlock(mmtl); } HRelease(BIF_P, hendp, hp); BIF_RET(previous); @@ -2630,12 +2746,30 @@ void init_db(void) size_t size; #ifdef ERTS_SMP - for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) { - erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i)); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, + (sizeof(erts_meta_main_tab_lock_t) + * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); + + if ((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) + meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) + ((((Uint) meta_main_tab_locks) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + + for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { + erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, + "meta_main_tab_slot", make_small(i)); } erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i)); + erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i)); } #endif @@ -2895,12 +3029,12 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb); /* while unlocked */ + db_ref(tb, LCK_NONE); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb); + tb = db_unref(tb, LCK_NONE); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -3008,12 +3142,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix)); + tb = db_ref(GET_ANY_SLOT_TAB(ix), LCK_WRITE); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; db_lock_take_over_ref(tb, LCK_WRITE); @@ -3045,7 +3180,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status |= DB_DELETE; if (is_atom(tb->common.id)) - remove_named_tab(tb->common.id); + remove_named_tab(tb, 0); free_heir_data(tb); free_fixations_locked(tb); @@ -3095,12 +3230,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb); + tb = db_ref(meta_main_tab[ix].u.tb, LCK_WRITE_REC); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int reds; DbFixation** pp; @@ -3274,6 +3410,7 @@ unlocked: #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ + db_ref(tb, LCK_WRITE); /* LCK_WRITE need it, but not LCK_READ */ erts_smp_rwmtx_runlock(&tb->common.rwlock); erts_smp_rwmtx_rwlock(&tb->common.rwlock); *kind_p = LCK_WRITE; @@ -3386,6 +3523,7 @@ static int free_table_cont(Process *p, int clean_meta_tab) { Eterm result; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG if (!first) { @@ -3411,9 +3549,20 @@ static int free_table_cont(Process *p, tb->common.id); #endif /* Completely done - we will not get called again. */ - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif free_slot(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); if (clean_meta_tab) { db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); @@ -3421,7 +3570,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb); + db_unref(tb, LCK_NONE); BUMP_REDS(p, 100); return 0; } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 124129a371..95ffc3afcd 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -623,12 +623,16 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; int i; + if (tb->common.type & DB_FREQ_READ) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i)); + erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", make_small(i)); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 382e5dceb5..672c5f2cd1 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -240,8 +240,9 @@ typedef struct db_table_common { #define DB_DUPLICATE_BAG (1 << 8) #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ +#define DB_FREQ_READ (1 << 11) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index aa37edafd1..d42820ddf3 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -186,10 +186,9 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_mutex_trylock()"); - return res; + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_trylock()"); + return ethr_mutex_trylock(&dmtx->mtx); #else return 0; #endif @@ -199,9 +198,9 @@ void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_lock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_lock()"); + ethr_mutex_lock(&dmtx->mtx); #endif } @@ -209,9 +208,9 @@ void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_unlock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_unlock()"); + ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -256,9 +255,9 @@ void erl_drv_cond_signal(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_signal()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_signal()"); + ethr_cond_signal(&dcnd->cnd); #endif } @@ -266,9 +265,9 @@ void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_broadcast()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_broadcast()"); + ethr_cond_broadcast(&dcnd->cnd); #endif } @@ -277,18 +276,13 @@ void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res; if (!dcnd || !dmtx) { - res = EINVAL; - error: - fatal_error(res, "erl_drv_cond_wait()"); + fatal_error(EINVAL, "erl_drv_cond_wait()"); } while (1) { - res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); + int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); if (res == 0) break; - if (res != EINTR) - goto error; } #endif } @@ -333,10 +327,9 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); + return ethr_rwmutex_tryrlock(&drwlck->rwmtx); #else return 0; #endif @@ -346,9 +339,9 @@ void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); + ethr_rwmutex_rlock(&drwlck->rwmtx); #endif } @@ -356,9 +349,9 @@ void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_runlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); + ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -366,10 +359,9 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrwlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); + return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #else return 0; #endif @@ -379,9 +371,9 @@ void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); + ethr_rwmutex_rwlock(&drwlck->rwmtx); #endif } @@ -389,9 +381,9 @@ void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwunlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); + ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 15d9538301..74e3ca048e 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock; #define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) #define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) #define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) -#define erts_fun_init_lock() erts_smp_rwmtx_init(&erts_fun_table_lock, \ - "fun_tab") static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -57,8 +55,12 @@ void erts_init_fun_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); - erts_fun_init_lock(); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4a4507b212..14bd10b42c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -78,6 +78,7 @@ static erts_tid_t main_thread; erts_cpu_info_t *erts_cpuinfo; +int erts_reader_groups; int erts_use_sender_punish; /* @@ -110,6 +111,7 @@ int erts_compat_rel; static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; +static int max_reader_groups; #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -505,6 +507,7 @@ void erts_usage(void) ERTS_MIN_COMPAT_REL, this_rel_num()); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); @@ -538,6 +541,50 @@ void erts_usage(void) erl_exit(-1, ""); } +#ifdef USE_THREADS +/* + * allocators for thread lib + */ + +static void *ethr_std_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size); +} +static void *ethr_std_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size); +} +static void ethr_std_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_STD, ptr); +} +static void *ethr_sl_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size); +} +static void *ethr_sl_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size); +} +static void ethr_sl_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_SL, ptr); +} +static void *ethr_ll_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size); +} +static void *ethr_ll_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size); +} +static void ethr_ll_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_LL, ptr); +} + +#endif + static void early_init(int *argc, char **argv) /* * Only put things here which are @@ -615,9 +662,15 @@ early_init(int *argc, char **argv) /* ? ncpuavail : (ncpuonln > 0 ? ncpuonln : no_schedulers)); +#ifdef ERTS_SMP + erts_max_main_threads = no_schedulers_online; +#endif + schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + max_reader_groups = ERTS_MAX_READER_GROUPS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -627,6 +680,24 @@ early_init(int *argc, char **argv) /* } if (argv[i][0] == '-') { switch (argv[i][1]) { + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + char *arg = get_arg(sub_param+1, argv[i+1], &i); + if (sscanf(arg, "%d", &max_reader_groups) != 1) { + erts_fprintf(stderr, + "bad reader groups limit: %s\n", arg); + erts_usage(); + } + if (max_reader_groups < 0) { + erts_fprintf(stderr, + "bad reader groups limit: %d\n", + max_reader_groups); + erts_usage(); + } + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -699,6 +770,36 @@ early_init(int *argc, char **argv) /* erts_early_init_scheduling(); /* Require allocators */ erts_init_utils(); /* Require allocators */ +#ifdef USE_THREADS + { + erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; + elid.mem.std.alloc = ethr_std_alloc; + elid.mem.std.realloc = ethr_std_realloc; + elid.mem.std.free = ethr_std_free; + elid.mem.sl.alloc = ethr_sl_alloc; + elid.mem.sl.realloc = ethr_sl_realloc; + elid.mem.sl.free = ethr_sl_free; + elid.mem.ll.alloc = ethr_ll_alloc; + elid.mem.ll.realloc = ethr_ll_realloc; + elid.mem.ll.free = ethr_ll_free; + +#ifdef ERTS_SMP + elid.main_threads = erts_max_main_threads; +#else + elid.main_threads = 1; +#endif + elid.reader_groups = (elid.main_threads > 1 + ? elid.main_threads + : 0); + if (max_reader_groups <= 1) + elid.reader_groups = 0; + if (elid.reader_groups > max_reader_groups) + elid.reader_groups = max_reader_groups; + erts_reader_groups = elid.reader_groups; + + erts_thr_late_init(&elid); + } +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif @@ -1193,9 +1294,17 @@ erl_start(int argc, char **argv) erts_async_thread_suggested_stack_size)); break; - case 'r': - erts_ets_realloc_always_moves = 1; + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + get_arg(sub_param+1, argv[i+1], &i); + /* already handled */ + } + else { + erts_ets_realloc_always_moves = 1; + } break; + } case 'n': /* XXX obsolete */ break; case 'c': @@ -1280,6 +1389,7 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else + erts_thr_set_main_status(1, 1); set_main_stack_size(); process_main(); #endif @@ -1353,7 +1463,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) && !defined(ERTS_SMP) +#if defined(USE_THREADS) exit_async(); #endif #if HAVE_ERTS_MSEG diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index cee470ae37..99cc80e259 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,10 +96,10 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, - { "db_tab", "address" }, - { "db_tab_fix", "address" }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, @@ -119,9 +119,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "child_status", NULL }, #endif #ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, + { "sys_driver_data_lock", NULL }, #endif - { "drv_ev_state_grow", NULL, }, + { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, @@ -153,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "alcu_delayed_free", "index" }, { "mseg", NULL }, #ifdef HALFWORD_HEAP { "pmmap", NULL }, @@ -177,17 +178,19 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "pix_lock", "address" }, { "run_queues_lists", NULL }, { "sched_stat", NULL }, + { "run_queue_sleep_list", "address" }, #endif { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP - { "proc_lck_wtr_alloc", NULL }, + { "proc_lck_qs_alloc", NULL }, #endif #ifdef __WIN32__ #ifdef DEBUG { "save_ops_lock", NULL }, #endif #endif - { "mtrace_buf", NULL } + { "mtrace_buf", NULL }, + { "erts_alloc_hard_debug", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -199,6 +202,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { & ERTS_LC_FLG_LT_ALL \ & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +static __decl_noreturn void __noreturn lc_abort(void); + static char * lock_type(Uint16 flags) { @@ -222,7 +227,7 @@ rw_op_str(Uint16 flags) return " (r)"; case ERTS_LC_FLG_LO_WRITE: erts_fprintf(stderr, "\nInternal error\n"); - abort(); + lc_abort(); default: break; } @@ -271,28 +276,18 @@ static erts_lc_free_block_t *free_blocks; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS static ethr_spinlock_t free_blocks_lock; -#define ERTS_LC_LOCK ethr_spin_lock -#define ERTS_LC_UNLOCK ethr_spin_unlock -#else -static ethr_mutex free_blocks_lock; -#define ERTS_LC_LOCK ethr_mutex_lock -#define ERTS_LC_UNLOCK ethr_mutex_unlock -#endif static ERTS_INLINE void lc_lock(void) { - if (ERTS_LC_LOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_lock(&free_blocks_lock); } static ERTS_INLINE void lc_unlock(void) { - if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_unlock(&free_blocks_lock); } static ERTS_INLINE void lc_free(void *p) @@ -313,7 +308,7 @@ static void *lc_core_alloc(void) { lc_unlock(); erts_fprintf(stderr, "Lock checker out of memory!\n"); - abort(); + lc_abort(); } #else @@ -327,7 +322,7 @@ static void *lc_core_alloc(void) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - abort(); + lc_abort(); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -367,11 +362,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - abort(); + lc_abort(); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; @@ -513,7 +508,7 @@ uninitialized_lock(void) { erts_fprintf(stderr, "Performing operations on uninitialized lock!\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } static void @@ -523,7 +518,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -533,7 +528,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -541,7 +536,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -550,7 +545,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) print_lock("Lock order violation occured when locking ", lck, "!\n"); print_curr_locks(l_lcks); print_lock_order(); - abort(); + lc_abort(); } static void @@ -561,7 +556,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, print_lock(op, lck, "!\n"); ASSERT(l_lcks); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -613,7 +608,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, } } print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -621,7 +616,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -629,7 +624,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l { print_lock("Unrequire on ", lck, " lock not required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -637,7 +632,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -645,7 +640,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } @@ -658,13 +653,23 @@ thread_exit_handler(void) erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } destroy_locked_locks(l_lcks); /* erts_tsd_set(locks_key, NULL); */ } } +static __decl_noreturn void +lc_abort(void) +{ +#ifdef __WIN32__ + DebugBreak(); +#else + abort(); +#endif +} + void erts_lc_set_thread_name(char *thread_name) { @@ -676,7 +681,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); } } @@ -686,7 +691,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion) erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n", file, line, assertion); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); return 0; } @@ -699,7 +704,7 @@ void erts_lc_fail(char *fmt, ...) va_end(args); erts_fprintf(stderr, "\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } @@ -719,7 +724,7 @@ erts_lc_get_lock_order_id(char *name) "(update erl_lock_check.c)\n", name); } - abort(); + lc_abort(); return (Sint16) -1; } @@ -895,6 +900,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } } +void +erts_lc_check_no_locked_of_type(Uint16 flags) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + if (l_lcks) { + erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { + if (l_lck->flags & flags) { + erts_fprintf(stderr, + "Locked lock of type %s found which isn't " + "allowed here!\n", + lock_type(l_lck->flags)); + print_curr_locks(l_lcks); + lc_abort(); + } + } + } +} + int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) { @@ -1283,13 +1307,8 @@ erts_lc_init(void) free_blocks = NULL; #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ -#ifdef ETHR_HAVE_NATIVE_LOCKS if (ethr_spinlock_init(&free_blocks_lock) != 0) - abort(); -#else - if (ethr_mutex_init(&free_blocks_lock) != 0) - abort(); -#endif + lc_abort(); erts_tsd_key_create(&locks_key); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index d5e2ede9ac..0372e6850d 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); +void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 26028aeefc..239773f366 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -166,8 +166,8 @@ static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char int i; type = lcnt_lock_type(lock->flag); - ethr_atomic_read(&lock->r_state, &r_state); - ethr_atomic_read(&lock->w_state, &w_state); + r_state = ethr_atomic_read(&lock->r_state); + w_state = ethr_atomic_read(&lock->w_state); if (lock->flag & flag) { @@ -394,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { ASSERT(eltd); - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); if (option & ERTS_LCNT_LO_WRITE) { - ethr_atomic_read(&lock->r_state, &r_state); + r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); } if (option & ERTS_LCNT_LO_READ) { @@ -423,7 +423,7 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc( &lock->w_state); eltd = lcnt_get_thread_data(); @@ -478,7 +478,7 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); } @@ -522,12 +522,12 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG /* flowstate */ - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 1); ethr_atomic_dec( &lock->flowstate); /* write state */ - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ASSERT(w_state > 0) #endif ethr_atomic_dec(&lock->w_state); @@ -558,7 +558,7 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { if (res != EBUSY) { #ifdef DEBUG - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); #endif diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 9cf55ee319..b1478758a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename) Uint16 port; erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_set_forksafe(&mtrace_buf_mutex); erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); - erts_mtx_set_forksafe(&mtrace_op_mutex); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 5865d33138..a3a1d95cd3 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) return dep_tmpl; @@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl) dep->prev = NULL; erts_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr); + erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; dep->connection_id = 0; @@ -580,6 +582,18 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) ErlNode ne; ne.sysname = sysname; ne.creation = creation; + + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + res = hash_get(&erts_node_table, (void *) &ne); + if (res && res != erts_this_node) { + long refc = erts_refc_inctest(&res->refc, 0); + if (refc < 2) /* New or pending delete */ + erts_refc_inc(&res->refc, 1); + } + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + if (res) + return res; + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); @@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; HashFunctions f; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; @@ -719,9 +737,10 @@ void erts_init_node_tables(void) erts_this_dist_entry->prev = NULL; erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, + &rwmtx_opt, + "dist_entry", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->sysname = am_Noname; erts_this_dist_entry->cid = NIL; erts_this_dist_entry->connection_id = 0; @@ -772,8 +791,8 @@ void erts_init_node_tables(void) (void) hash_put(&erts_node_table, (void *) erts_this_node); - erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table"); - erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table"); + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); references_atoms_need_init = 1; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 967a14f0d1..c10724b951 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id, if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); + erts_smp_runq_unlock(runq); } else { enqueue_port(runq, pp); @@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id, profile_runnable_port(pp, am_active); } + erts_smp_runq_unlock(runq); + erts_smp_notify_inc_runq(runq); } - erts_smp_runq_unlock(runq); return 0; } @@ -944,8 +946,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; - erts_smp_notify_inc_runq(xrunq); erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; @@ -1112,7 +1114,6 @@ erts_port_migrate(Port *prt, int *prt_locked, dequeue_port(from_rq, prt); erts_smp_atomic_set(&prt->run_queue, (long) to_rq); enqueue_port(to_rq, prt); - erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 41031f5468..abf4c9d96a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -46,7 +46,13 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 +#define ERTS_SCHED_SPIN_UNTIL_YIELD 100 + +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ + (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) +#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 #define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) #define ERTS_WAKEUP_OTHER_DEC 10 @@ -106,6 +112,10 @@ Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; +#ifdef ERTS_SMP +Uint erts_max_main_threads; +#endif + int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -116,16 +126,34 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHED_CHANGING_ONLINE 1 -#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((long) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((long) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((long) 1) << 2) + +#ifndef DEBUG + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic_set(&schdlr_sspnd.changing, (VAL)) + +#else + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ +do { \ + long old_val__ = erts_smp_atomic_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) + +#endif + static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int changing; int online; int curr_online; int wait_curr_online; + erts_smp_atomic_t changing; erts_smp_atomic_t active; struct { erts_smp_atomic_t ongoing; @@ -231,6 +259,17 @@ typedef union { ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +#ifdef ERTS_SMP + +typedef union { + ErtsSchedulerSleepInfo ssi; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; +} ErtsAlignedSchedulerSleepInfo; + +static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; + +#endif + #ifndef BM_COUNTERS static int processes_busy; #endif @@ -288,8 +327,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -353,6 +399,11 @@ static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size #endif +static int reader_group_lookup(int logical); +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + static void early_cpu_bind_init(void); static void late_cpu_bind_init(void); @@ -582,6 +633,76 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +void +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, long flags) +{ + switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_sys_schedule_interrupt(1); + break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_tse_set(ssi->event); + break; + case 0: + break; + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + __FILE__, __LINE__); + break; + } +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + + for (i = 0; i < erts_no_schedulers; i++) { + long aux_work; + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(i); + aux_work = erts_smp_atomic_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) + erts_sched_poke(ssi); + } +} +#endif + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +static ERTS_INLINE long +blockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = erts_smp_atomic_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; + erts_check_children(); + } +#endif + } + return aux_work; +} + +#endif + +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +static ERTS_INLINE long +nonblockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + + } +} +#endif + static void prepare_for_block(void *vrq) { @@ -634,7 +755,31 @@ erts_active_schedulers(void) return as; } +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ #ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + erts_smp_atomic_set(&doing_sys_schedule, 0); + } + return 0; +#else + return !erts_port_task_have_outstanding_io_tasks(); +#endif +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; +} static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) @@ -695,101 +840,304 @@ non_empty_runq(ErtsRunQueue *rq) } } -static ERTS_INLINE int -sched_spin_wake(ErtsRunQueue *rq) +static long +sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = 0; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_atomic_inc(&rq->spin_wake); - return 1; - } - return 0; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED)); + return oflgs; } -static ERTS_INLINE int -sched_spin_wake_all(ErtsRunQueue *rq) +static long +sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = ERTS_SSI_FLG_WAITING; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) - erts_smp_atomic_add(&rq->spin_wake, val); - return val; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } while (oflgs & ERTS_SSI_FLG_WAITING); + return oflgs; +} + +static long +sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + long until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; } +static long +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, long sleep_type) +{ + long oflgs; + long nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + + if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + return oflgs; + } + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } +} + +#define ERTS_SCHED_WAIT_WOKEN(FLGS) \ + (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ + != ERTS_SSI_FLG_WAITING) + static void -sched_sys_wait(Uint no, ErtsRunQueue *rq) +scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { - long dt; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + int spincount; + long flgs; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_smp_spin_lock(&rq->sleepers.lock); + flgs = sched_prep_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + /* Go suspend instead... */ + erts_smp_spin_unlock(&rq->sleepers.lock); + return; + } + + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + + /* + * If all schedulers are waiting, one of them *should* + * be waiting in erl_sys_schedule() + */ + + if (!prepare_for_sys_schedule()) { + + sched_waiting(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + tse_wait: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + tse_blockable_aux_work: + aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - sched_waiting_sys(no, rq); + while (1) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif - erl_sys_schedule(1); /* Might give us something to do */ + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + + flgs = sched_prep_cont_spin_wait(ssi); + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto tse_blockable_aux_work; + } +#endif - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_runq_unlock(rq); } - } - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + erts_smp_runq_lock(rq); + sched_active(esdp->no, rq); + } else { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + long dt; + + erts_smp_atomic_set(&function_calls, 0); + *fcalls = 0; + + sched_waiting_sys(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + + while (spincount-- > 0) { + + sys_poll_aux_work: + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); + + sys_aux_work: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + } + + /* + * If we got new I/O tasks we aren't allowed to + * call erl_sys_schedule() until it is handled. + */ + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to continue checking for I/O... + */ + if (!prepare_for_sys_schedule()) { + spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + goto tse_wait; + } + } + } + + erts_smp_runq_lock(rq); + /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). */ - if (!erts_port_task_have_outstanding_io_tasks()) { -#endif + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to wait in erl_sys_schedule() after all... + */ + if (prepare_for_sys_schedule()) + goto do_sys_schedule; + + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + else { + do_sys_schedule: erts_sys_schedule_interrupt(0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } + + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_smp_runq_unlock(rq); erl_sys_schedule(0); @@ -797,134 +1145,103 @@ sched_sys_wait(Uint no, ErtsRunQueue *rq) dt = do_time_read_and_reset(); if (dt) bump_timer(dt); - erts_smp_runq_lock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + erts_smp_atomic_set(&doing_sys_schedule, 0); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + sched_active_sys(esdp->no, rq); } } -#endif - sched_active_sys(no, rq); -} - -static void -sched_cnd_wait(Uint no, ErtsRunQueue *rq) -{ -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif - - sched_waiting(no, rq); - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); -#else - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_mtx_unlock(&rq->mtx); - - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_mtx_unlock(&rq->mtx); - } - } - - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val == 0) { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); - } - else { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - } -#endif - - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - - sched_active(no, rq); } -static void -wake_one_scheduler(void) -{ - ASSERT(erts_common_run_queue); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); - if (erts_common_run_queue->waiting) { - if (!sched_spin_wake(erts_common_run_queue)) { - if (erts_common_run_queue->waiting == -1) /* One scheduler waiting - and doing so in - sys_schedule */ - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&erts_common_run_queue->cnd); - } +static ERTS_INLINE long +ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) +{ + /* reset all flags but suspended */ + long oflgs; + long nflgs = 0; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return oflgs; + nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + xflgs = oflgs; } } static void -wake_scheduler(ErtsRunQueue *rq, int incq) +wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - ASSERT(!erts_common_run_queue); - ASSERT(-1 <= rq->waiting && rq->waiting <= 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->waiting && !rq->woken) { - if (!sched_spin_wake(rq)) { - if (rq->waiting < 0) - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&rq->cnd); + int res; + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; + + /* + * The unlocked run queue is not strictly necessary + * from a thread safety or deadlock prevention + * perspective. It will, however, cost us performance + * if it is locked during wakup of another scheduler, + * so all code *should* handle this without having + * the lock on the run queue. + */ + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + sl = &rq->sleepers; + + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) + erts_smp_spin_unlock(&sl->lock); + else if (one) { + long flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; } - rq->woken = 1; - if (incq) + if (ssi->next) + ssi->next->prev = ssi->prev; + + res = sl->list != NULL; + erts_smp_spin_unlock(&sl->lock); + + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + + if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) non_empty_runq(rq); } + else { + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } static void wake_all_schedulers(void) { - if (erts_common_run_queue) { - erts_smp_runq_lock(erts_common_run_queue); - if (erts_common_run_queue->waiting) { - if (erts_common_run_queue->waiting < 0) - erts_sys_schedule_interrupt(1); - sched_spin_wake_all(erts_common_run_queue); - erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); - } - erts_smp_runq_unlock(erts_common_run_queue); - } + if (erts_common_run_queue) + wake_scheduler(erts_common_run_queue, 0, 0); else { int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - wake_scheduler(rq, 0); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); } } } @@ -939,14 +1256,14 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) wrq = ERTS_RUNQ_IX(ix); iflgs = erts_smp_atomic_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { - erts_smp_xrunq_lock(crq, wrq); if (activate) { if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0); - erts_smp_xrunq_unlock(crq, wrq); + wake_scheduler(wrq, 0, 1); return 1; } return 0; @@ -992,15 +1309,13 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (erts_common_run_queue) - wake_one_scheduler(); - else - wake_scheduler(runq, 1); + if (runq) + wake_scheduler(runq, 1, 1); #endif } void -erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +erts_smp_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } @@ -1146,20 +1461,23 @@ static void evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) { Port *prt; + int notify_to_rq = 0; int prio; int prt_locked = 0; int rq_locked = 0; int evac_rq_locked = 1; + ErtsMigrateResult mres; erts_smp_runq_lock(evac_rq); + erts_smp_atomic_bor(&evac_rq->scheduler->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1187,9 +1505,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) /* Evacuate scheduled ports */ prt = evac_rq->ports.start; while (prt) { - (void) erts_port_migrate(prt, &prt_locked, + mres = erts_port_migrate(prt, &prt_locked, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (prt_locked) erts_smp_port_unlock(prt); if (!evac_rq_locked) { @@ -1218,9 +1538,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) goto end_of_proc; } - (void) erts_proc_migrate(proc, &proc_locks, + mres = erts_proc_migrate(proc, &proc_locks, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); if (!evac_rq_locked) { @@ -1252,10 +1574,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (rq_locked) erts_smp_runq_unlock(rq); - if (!evac_rq_locked) - erts_smp_runq_lock(evac_rq); - wake_scheduler(evac_rq, 0); - erts_smp_runq_unlock(evac_rq); + if (evac_rq_locked) + erts_smp_runq_unlock(evac_rq); + + if (notify_to_rq) + smp_notify_inc_runq(rq); + + wake_scheduler(evac_rq, 0, 1); } static int @@ -1483,31 +1808,6 @@ try_steal_task(ErtsRunQueue *rq) return res; } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) { - erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); - ERTS_SCHEDULER_IX(i)->check_children = 1; - if (!erts_common_run_queue) - wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); - erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); - } - if (ongoing_multi_scheduling_block()) { - /* Also blocked schedulers need to check children */ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - for (i = 0; i < erts_no_schedulers; i++) - ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } - if (erts_common_run_queue) - wake_all_schedulers(); -} -#endif - /* Run queue balancing */ typedef struct { @@ -2100,6 +2400,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_atomic_init(&no_empty_run_queues, 0); #endif + erts_no_run_queues = n; + for (ix = 0; ix < n; ix++) { int pix, rix; ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); @@ -2114,8 +2416,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); - erts_smp_atomic_init(&rq->spin_waiter, 0); - erts_smp_atomic_init(&rq->spin_wake, 0); +#ifdef ERTS_SMP + erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif rq->waiting = 0; rq->woken = 0; @@ -2166,7 +2470,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - erts_no_run_queues = n; #ifdef ERTS_SMP @@ -2181,9 +2484,34 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #endif + n = (int) no_schedulers; + erts_no_schedulers = n; + +#ifdef ERTS_SMP + /* Create and initialize scheduler sleep info */ + + aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, + (sizeof(ErtsAlignedSchedulerSleepInfo) + *(n+1))); + if ((((Uint) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) + aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) + ((((Uint) aligned_sched_sleep_info) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + for (ix = 0; ix < n; ix++) { + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); +#if 0 /* no need to initialize these... */ + ssi->next = NULL; + ssi->prev = NULL; +#endif + erts_smp_atomic_init(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_thread_func */ + erts_smp_atomic_init(&ssi->aux_work, 0); + } +#endif + /* Create and initialize scheduler specific data */ - n = (int) no_schedulers; erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, (sizeof(ErtsAlignedSchedulerData) *(n+1))); @@ -2200,6 +2528,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; #if HALFWORD_HEAP /* Registers need to be heap allocated (correct memory range) for tracing to work */ @@ -2228,11 +2557,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->check_children = 0; - esdp->blocked_check_children = 0; -#endif - erts_smp_atomic_init(&esdp->suspended, 0); erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2241,7 +2565,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; + erts_smp_atomic_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); @@ -2264,7 +2588,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2275,7 +2600,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.wait_curr_online = no_schedulers_online; schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); erts_smp_atomic_init(&doing_sys_schedule, 0); @@ -2423,13 +2749,115 @@ susp_sched_resume_block(void *unused) } static void +scheduler_ix_resume_wake(Uint ix) +{ + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long oflgs; + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, 0, xflgs); + if (oflgs == xflgs) { + erts_sched_finish_poke(ssi, oflgs); + break; + } + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); +} + +static long +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, long xpct) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = xpct; + + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + + return oflgs; +} + +static long +sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static long +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + return oflgs; + } + xflgs = oflgs; + } +} + +static void suspend_scheduler(ErtsSchedulerData *esdp) { + long flgs; + int changing; long no = (long) esdp->no; ErtsRunQueue *rq = esdp->run_queue; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; + int reset_read_group = 0; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif /* * Schedulers may be suspended in two different ways: @@ -2451,113 +2879,151 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (scheduler2cpu_map[esdp->no].bound_id >= 0 && erts_unbind_from_cpu(erts_cpuinfo) == 0) { esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + reset_read_group = 1; } erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (reset_read_group) + erts_smp_rwmtx_set_reader_group(0); + + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(0, 0); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) - schdlr_sspnd.changing = 0; - } - - while (1) { + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; - erts_smp_runq_lock(esdp->run_queue); - check_children = esdp->check_children; - esdp->check_children = 0; - erts_smp_runq_unlock(esdp->run_queue); - if (check_children) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_check_children(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } } -#endif - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { - int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; - curr_online = 0; - changed = 1; + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; - curr_online = 1; - changed = 1; + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; } - if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) - schdlr_sspnd.changing = 0; - } - if (wake) { - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - wake = 0; - } + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + blockable_aux_work: + blockable_aux_work(esdp, ssi, aux_work); +#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (1) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + while (1) { + long flgs; +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->blocked_check_children) - break; + flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; + + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto blockable_aux_work; + } #endif - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + } + + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) - break; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + } - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; + active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && schdlr_sspnd.online == active_schedulers) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->blocked_check_children = 0; -#endif + ASSERT(no <= schdlr_sspnd.online); + ASSERT(!erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); } - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED - && schdlr_sspnd.online == active_schedulers) { - schdlr_sspnd.changing = 0; - } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(curr_online); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -2622,8 +3088,10 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; + long changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (yield_allowed && schdlr_sspnd.changing) + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { *active = *online = schdlr_sspnd.online; @@ -2643,6 +3111,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; + long changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -2652,7 +3121,8 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } else { @@ -2661,17 +3131,19 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); schdlr_sspnd.online = no; if (no > online) { int ix; schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) - /* No schedulers to resume */; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } else if (erts_common_run_queue) { for (ix = online; ix < no; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 0); + scheduler_ix_resume_wake(ix); } else { if (plocks) { @@ -2685,6 +3157,7 @@ erts_set_schedulers_online(Process *p, erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* * Spread evacuation paths among all online @@ -2699,7 +3172,6 @@ erts_set_schedulers_online(Process *p, erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); res = ERTS_SCHDLR_SSPND_DONE; } else /* if (no < online) */ { @@ -2716,12 +3188,17 @@ erts_set_schedulers_online(Process *p, schdlr_sspnd.wait_curr_online = no+1; } - if (ongoing_multi_scheduling_block()) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - else if (erts_common_run_queue) { + if (ongoing_multi_scheduling_block()) { for (ix = no; ix < online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 1); + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) { + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } wake_all_schedulers(); } else { @@ -2748,7 +3225,10 @@ erts_set_schedulers_online(Process *p, erts_smp_atomic_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); + } } } @@ -2762,6 +3242,12 @@ erts_set_schedulers_online(Process *p, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -2776,11 +3262,12 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; + long changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } else if (on) { /* ------ BLOCK ------ */ @@ -2794,19 +3281,22 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { + int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + ASSERT(0 == erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); - if (schdlr_sspnd.online == 1) { + if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); if (p->scheduler_data->no == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 1; @@ -2820,17 +3310,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + for (ix = 1; ix < online; ix++) + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); erts_smp_atomic_set(&balance_info.used_runqs, 1); - for (ix = 0; ix < schdlr_sspnd.online; ix++) { + for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); erts_smp_runq_unlock(rq); } @@ -2855,6 +3347,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -2898,7 +3397,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.msb.procs) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); #ifdef DEBUG ERTS_FOREACH_RUNQ(rq, { @@ -2925,13 +3424,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); - schdlr_sspnd.changing = 0; + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + erts_smp_atomic_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2948,6 +3447,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* Spread evacuation paths among all online run queues */ @@ -2963,7 +3463,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -3035,18 +3534,34 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { +#ifdef ERTS_SMP + Uint no = ((ErtsSchedulerData *) vesdp)->no; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; - Uint no = ((ErtsSchedulerData *) vesdp)->no; erts_snprintf(&buf[0], 31, "scheduler %bpu", no); erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP + + if (no <= erts_max_main_threads) { + erts_thr_set_main_status(1, (int) no); + if (erts_reader_groups) { + int rg = (int) no; + if (rg > erts_reader_groups) + rg = (((int) no) - 1) % erts_reader_groups + 1; + erts_smp_rwmtx_set_reader_group(rg); + } + } + erts_proc_lock_prepare_proc_lock_waiter(); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + + #endif erts_register_blockable_thread(); #ifdef HIPE @@ -3055,30 +3570,30 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); - schdlr_sspnd.curr_online--; + if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + } - if (((ErtsSchedulerData *) vesdp)->no != 1) { - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - schdlr_sspnd.changing = 0; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); } - } - else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - schdlr_sspnd.changing = 0; - else { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - ASSERT(!schdlr_sspnd.changing); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3428,6 +3943,7 @@ processor_order_cmp(const void *vx, const void *vy) static void check_cpu_bind(ErtsSchedulerData *esdp) { + int rg = 0; int res; int cpu_id; erts_smp_runq_unlock(esdp->run_queue); @@ -3459,6 +3975,12 @@ check_cpu_bind(ErtsSchedulerData *esdp) erts_send_error_to_logger_nogl(dsbufp); } } + if (erts_reader_groups) { + if (esdp->cpu_id >= 0) + rg = reader_group_lookup(esdp->cpu_id); + else + rg = (((int) esdp->no) - 1) % erts_reader_groups + 1; + } erts_smp_runq_lock(esdp->run_queue); #ifdef ERTS_SMP if (erts_common_run_queue) @@ -3469,6 +3991,8 @@ check_cpu_bind(ErtsSchedulerData *esdp) #endif erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (erts_reader_groups) + erts_smp_rwmtx_set_reader_group(rg); } static void @@ -3497,11 +4021,13 @@ signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) wake_all_schedulers(); } else { - ERTS_FOREACH_RUNQ(rq, - { + for (s_ix = 0; s_ix < erts_no_run_queues; s_ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(s_ix); + erts_smp_runq_lock(rq); rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - wake_scheduler(rq, 0); - }); + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; } #else check_cpu_bind(erts_get_scheduler_data()); @@ -3541,6 +4067,490 @@ erts_init_scheduler_bind_type(char *how) return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; } +/* + * reader groups map + */ + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int *map; + int size; + int groups; +} erts_reader_groups_map_test; + +typedef struct { + int id; + int sub_levels; + int reader_groups; +} erts_rg_count_t; + +typedef struct { + int logical; + int reader_group; +} erts_reader_groups_map_t; + +typedef struct { + erts_reader_groups_map_t *map; + int map_size; + int logical_processors; + int groups; +} erts_make_reader_groups_map_test; + +static int reader_groups_available_cpu_check; +static int reader_groups_logical_processors; +static int reader_groups_map_size; +static erts_reader_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_RG ERTS_TOPOLOGY_MAX_DEPTH + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test); + +static Eterm +get_reader_groups_map(Process *c_p, + erts_reader_groups_map_t *map, + int map_size, + int logical_processors) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + logical_processors*(2+3); +#endif + for (i = map_size - 1; i >= 0; i--) { + if (map[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map[i].logical), + make_small(map[i].reader_group)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_make_reader_groups_map_test test; + + test.groups = groups; + make_reader_groups_map(&test); + if (!test.map) + res = NIL; + else { + res = get_reader_groups_map(c_p, + test.map, + test.map_size, + test.logical_processors); + erts_free(ERTS_ALC_T_TMP, test.map); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + res = get_reader_groups_map(c_p, + reader_groups_map, + reader_groups_map_size, + reader_groups_logical_processors); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(erts_cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_RG] = 0; + + ASSERT(last.logical != cpudata[a].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static int +reader_group_lookup(int logical) +{ + int start = logical % reader_groups_map_size; + int ix = start; + + do { + if (reader_groups_map[ix].logical == logical) { + ASSERT(reader_groups_map[ix].reader_group > 0); + return reader_groups_map[ix].reader_group; + } + ix++; + if (ix == reader_groups_map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +reader_group_insert(erts_reader_groups_map_t *map, int map_size, + int logical, int reader_group) +{ + int start = logical % map_size; + int ix = start; + + do { + if (map[ix].logical < 0) { + map[ix].logical = logical; + map[ix].reader_group = reader_group; + return; + } + ix++; + if (ix == map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_rg_count_t *rgc, int level, int aix, int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + rgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + rgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && rgc->id == avail[aix].level[level]); + rgc->reader_groups = 0; + return aix; +} + +static int +write_reader_groups(int *rgp, erts_rg_count_t *rgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int rg = *rgp; + int sub_level = level+1; + int sl_per_gr = rgcp->sub_levels / rgcp->reader_groups; + int xsl = rgcp->sub_levels % rgcp->reader_groups; + int sls = 0; + int last = -1; + int xsl_rg_lim = (rgcp->reader_groups - xsl) + rg + 1; + + ASSERT(level < 0 || avail[a].level[level] == rgcp->id) + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + rg++; + if (rg >= xsl_rg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_RG] = rg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == rgcp->id)); + + ASSERT(rgcp->reader_groups == rg - *rgp); + + *rgp = rg; + + return a; +} + +static int +rg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +rg_count_id_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + return x->id - y->id; +} + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + erts_reader_groups_map_t *map; + int map_sz; + int groups = erts_reader_groups; + + if (test) { + test->map = NULL; + test->map_size = 0; + groups = test->groups; + } + + if (!groups) + return; + + if (!test) { + if (reader_groups_map) + erts_free(ERTS_ALC_T_RDR_GRPS_MAP, reader_groups_map); + + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; + } + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test != NULL); + + destroy_tmp_cpu_topology_copy(cpudata); + + map_sz = avail_sz*2+1; + + if (test) { + map = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + test->map = map; + test->map_size = map_sz; + test->logical_processors = avail_sz; + } + else { + map = erts_alloc(ERTS_ALC_T_RDR_GRPS_MAP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + reader_groups_map = map; + reader_groups_logical_processors = avail_sz; + reader_groups_map_size = map_sz; + + } + + for (i = 0; i < map_sz; i++) { + map[i].logical = -1; + map[i].reader_group = 0; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= groups) { + int a, rg, last = -1; + rg = 0; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + rg++; + last = avail[a].level[spread_level]; + } + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + rg); + } + } + else { /* groups < no.level[spread_level] */ + erts_rg_count_t *rg_count; + int a, rg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + rg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_rg_count_t)); + + if (toplevels == 1) { + rg_count[0].id = 0; + rg_count[0].sub_levels = no.level[spread_level]; + rg_count[0].reader_groups = groups; + } + else { + int rgs_per_tl, rgs; + rgs = groups; + rgs_per_tl = rgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + rg_count[i].id = avail[a].level[tl]; + a = sub_levels(&rg_count[i], tl, a, avail_sz, avail); + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels < rgs_per_tl) { + rg_count[i].reader_groups = rg_count[i].sub_levels; + rgs -= rg_count[i].sub_levels; + } + else { + rg_count[i].reader_groups = rgs_per_tl; + rgs -= rgs_per_tl; + } + } + + while (rgs > 0) { + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels == rg_count[i].reader_groups) + break; + else { + rg_count[i].reader_groups++; + if (--rgs == 0) + break; + } + } + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_id_compare); + } + + a = i = rg = 0; + while (a < avail_sz) { + a = write_reader_groups(&rg, &rg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(groups == rg); + + for (a = 0; a < avail_sz; a++) + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_RG]); + + erts_free(ERTS_ALC_T_TMP, rg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +/* + * CPU topology + */ + typedef struct { int *id; int used; @@ -4054,6 +5064,8 @@ erts_set_cpu_topology(Process *c_p, Eterm term) sizeof(erts_cpu_topology_t)*cpudata_size); } + make_reader_groups_map(NULL); + signal_schedulers_bind_change(cpudata, cpudata_size); done: @@ -4457,6 +5469,11 @@ early_cpu_bind_init(void) cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + reader_groups_available_cpu_check = 1; + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; + if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, system_cpudata_size)) { @@ -4492,6 +5509,8 @@ late_cpu_bind_init(void) : ERTS_CPU_BIND_NONE); } + make_reader_groups_map(NULL); + if (cpu_bind_order != ERTS_CPU_BIND_NONE) { erts_cpu_topology_t *cpudata; int cpudata_size; @@ -5357,7 +6376,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p) } /* schedule a process */ -static ERTS_INLINE void +static ERTS_INLINE ErtsRunQueue * internal_add_to_runq(ErtsRunQueue *runq, Process *p) { Uint32 prev_status = p->status; @@ -5368,12 +6387,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return; + return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { ASSERT(p->status != P_SUSPENDED); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return; + return NULL; } ASSERT(!p->scheduler_data); #endif @@ -5412,20 +6431,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) profile_runnable_proc(p, am_active); } - smp_notify_inc_runq(add_runq); - if (add_runq != runq) erts_smp_runq_unlock(add_runq); + + return add_runq; } void erts_add_to_runq(Process *p) { + ErtsRunQueue *notify_runq; ErtsRunQueue *runq = erts_get_runq_proc(p); erts_smp_runq_lock(runq); - internal_add_to_runq(runq, p); + notify_runq = internal_add_to_runq(runq, p); erts_smp_runq_unlock(runq); + smp_notify_inc_runq(notify_runq); + } /* Possibly remove a scheduled process we need to suspend */ @@ -5564,8 +6586,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); - smp_notify_inc_runq(to_rq); - return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -5762,30 +6782,6 @@ erts_set_process_priority(Process *p, Eterm new_value) return old_value; } -#ifdef ERTS_SMP - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - return 0; -} - -#else - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - return !erts_port_task_have_outstanding_io_tasks(); -} - -#endif - /* note that P_RUNNING is only set so that we don't try to remove ** running processes from the schedule queue if they exit - a running ** process not being in the schedule queue!! @@ -5920,8 +6916,11 @@ Process *schedule(Process *p, int calls) p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + ErtsRunQueue *notify_runq; p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); + if (notify_runq != rq) + smp_notify_inc_runq(notify_runq); } #endif @@ -5989,7 +6988,10 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) @@ -5998,12 +7000,21 @@ Process *schedule(Process *p, int calls) } } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->check_children) { - esdp->check_children = 0; - erts_smp_runq_unlock(rq); - erts_check_children(); - erts_smp_runq_lock(rq); +#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) + { + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + long aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work) { + erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + erts_smp_runq_lock(rq); + } } #endif @@ -6035,7 +7046,10 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; } @@ -6052,17 +7066,7 @@ Process *schedule(Process *p, int calls) } } - if (prepare_for_sys_schedule()) { - erts_smp_atomic_set(&function_calls, 0); - fcalls = 0; - sched_sys_wait(esdp->no, rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - else { - /* If all schedulers are waiting, one of them *should* - be waiting in erl_sys_schedule() */ - sched_cnd_wait(esdp->no, rq); - } + scheduler_wait(&fcalls, esdp, rq); non_empty_runq(rq); @@ -6124,7 +7128,7 @@ Process *schedule(Process *p, int calls) else { if (erts_common_run_queue) { if (erts_common_run_queue->waiting) - wake_one_scheduler(); + wake_scheduler(erts_common_run_queue, 0, 1); } else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); @@ -6439,8 +7443,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; - smp_notify_inc_runq(rq); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } static void @@ -6682,7 +7686,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq; + ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ #ifndef HYBRID @@ -6999,10 +8003,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif p->status = P_WAITING; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(notify_runq); + res = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 8f9f7f004e..6d7de237f3 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -89,6 +89,7 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP +extern Uint erts_max_main_threads; #include "erl_bits.h" #endif @@ -219,6 +220,51 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; +#define ERTS_SSI_FLG_SLEEPING (((long) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((long) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((long) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((long) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((long) 1) << 4) + +#define ERTS_SSI_FLGS_SLEEP_TYPE \ + (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) + +#define ERTS_SSI_FLGS_SLEEP \ + (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE) + +#define ERTS_SSI_FLGS_ALL \ + (ERTS_SSI_FLGS_SLEEP \ + | ERTS_SSI_FLG_WAITING \ + | ERTS_SSI_FLG_SUSPENDED) + + +#if !defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + && defined(ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN) +#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +#endif + +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((long) 1) << 0) + +#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN) +#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ + (0) + +typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; + +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; + +struct ErtsSchedulerSleepInfo_ { + ErtsSchedulerSleepInfo *next; + ErtsSchedulerSleepInfo *prev; + erts_smp_atomic_t flags; + erts_tse_t *event; + erts_smp_atomic_t aux_work; +}; + /* times to reschedule low prio process before running */ #define RESCHEDULE_LOW 8 @@ -271,8 +317,9 @@ struct ErtsRunQueue_ { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - erts_smp_atomic_t spin_waiter; - erts_smp_atomic_t spin_wake; +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -353,6 +400,7 @@ struct ErtsSchedulerData_ { ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ + ErtsSchedulerSleepInfo *ssi; Process *free_process; #endif #if !HEAP_ON_C_STACK @@ -374,11 +422,6 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; /* run queue mutex */ - int blocked_check_children; /* schdlr_sspnd mutex */ -#endif - erts_smp_atomic_t suspended; /* Only used when common run queue */ erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ #endif }; @@ -1085,6 +1128,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); void erts_deep_process_dump(int, void *); +Eterm erts_get_reader_groups_map(Process *c_p); +Eterm erts_debug_reader_groups_map(Process *c_p, int groups); + Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); @@ -1509,29 +1555,30 @@ extern int erts_disable_proc_not_running_opt; #define ERTS_MIN_PROCESSES 16 #endif -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_smp_notify_inc_runq__(ErtsRunQueue *runq); -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ +void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#ifdef ERTS_SMP +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, long); +ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (runq->waiting) - erts_smp_notify_inc_runq__(runq); -#endif + long flags = erts_smp_atomic_read(&ssi->flags); + ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) + || (flags & ERTS_SSI_FLG_WAITING)); + if (flags & ERTS_SSI_FLG_SLEEPING) { + flags = erts_smp_atomic_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + erts_sched_finish_poke(ssi, flags); + } } -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #ifdef ERTS_SMP */ + #include "erl_process_lock.h" #undef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 52440fb635..a4d12139e9 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -71,9 +71,12 @@ const Process erts_proc_lock_busy; #ifdef ERTS_SMP -/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/ -#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 16000 +#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 +#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 +#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50 + +#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25 #ifdef ERTS_PROC_LOCK_DEBUG #define ERTS_PROC_LOCK_HARD_DEBUG @@ -83,32 +86,19 @@ const Process erts_proc_lock_busy; static void check_queue(erts_proc_lock_t *lck); #endif - -typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t; -struct erts_proc_lock_waiter_t_ { - erts_proc_lock_waiter_t *next; - erts_proc_lock_waiter_t *prev; - ErtsProcLocks wait_locks; - erts_smp_gate_t gate; - erts_proc_lock_queues_t *queues; -}; +#if SIZEOF_INT < 4 +#error "The size of the 'uflgs' field of the erts_tse_t type is too small" +#endif struct erts_proc_lock_queues_t_ { erts_proc_lock_queues_t *next; - erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -}; - -struct erts_proc_lock_thr_spec_data_t_ { - erts_proc_lock_queues_t *qs; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; }; static erts_proc_lock_queues_t zeroqs = {0}; -static erts_smp_spinlock_t wtr_lock; -static erts_proc_lock_waiter_t *waiter_free_list; +static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; -static erts_tsd_key_t waiter_key; #ifdef ERTS_ENABLE_LOCK_CHECK static struct { @@ -122,35 +112,26 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; static int proc_lock_spin_count; -static int proc_lock_trans_spin_cost; +static int aux_thr_proc_lock_spin_count; -static void cleanup_waiter(void); +static void cleanup_tse(void); void erts_init_proc_lock(void) { int i; int cpus; - erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc"); + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#if ERTS_PROC_LOCK_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i)); -#else - erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif -#else #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i)); + erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif -#endif } - waiter_free_list = NULL; queue_free_list = NULL; - erts_tsd_key_create(&waiter_key); - erts_thr_install_exit_handler(cleanup_waiter); + erts_thr_install_exit_handler(cleanup_tse); #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); @@ -158,86 +139,106 @@ erts_init_proc_lock(void) lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif cpus = erts_get_cpu_configured(erts_cpuinfo); - if (cpus > 1) - proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE - * ((int) erts_no_schedulers)); - else if (cpus == 1) - proc_lock_spin_count = 0; - else /* No of cpus unknown. Assume multi proc, but be conservative. */ + if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; - if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) - proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; - proc_lock_trans_spin_cost = proc_lock_spin_count/20; -} - -static ERTS_INLINE erts_proc_lock_waiter_t * -alloc_wtr(void) -{ - erts_proc_lock_waiter_t *wtr; - erts_smp_spin_lock(&wtr_lock); - wtr = waiter_free_list; - if (wtr) { - waiter_free_list = wtr->next; - ERTS_LC_ASSERT(queue_free_list); - wtr->queues = queue_free_list; - queue_free_list = wtr->queues->next; - erts_smp_spin_unlock(&wtr_lock); + proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC + * ((int) erts_no_schedulers)); + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT; } - else { - erts_smp_spin_unlock(&wtr_lock); - wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR, - sizeof(erts_proc_lock_waiter_t)); - erts_smp_gate_init(&wtr->gate); - wtr->wait_locks = (ErtsProcLocks) 0; - wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, - sizeof(erts_proc_lock_queues_t)); - sys_memcpy((void *) wtr->queues, - (void *) &zeroqs, - sizeof(erts_proc_lock_queues_t)); + else if (cpus == 1) { + proc_lock_spin_count = 0; + aux_thr_proc_lock_spin_count = 0; } - return wtr; + else { /* No of cpus unknown. Assume multi proc, but be conservative. */ + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2; + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2; + } + if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; } #ifdef ERTS_ENABLE_LOCK_CHECK static void -check_unused_waiter(erts_proc_lock_waiter_t *wtr) +check_unused_tse(erts_tse_t *wtr) { int i; - ERTS_LC_ASSERT(wtr->wait_locks == 0); + erts_proc_lock_queues_t *queues = wtr->udata; + ERTS_LC_ASSERT(wtr->uflgs == 0); for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - ERTS_LC_ASSERT(!wtr->queues->queue[i]); + ERTS_LC_ASSERT(!queues->queue[i]); } -#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W)) +#define CHECK_UNUSED_TSE(W) check_unused_tse((W)) #else -#define CHECK_UNUSED_WAITER(W) +#define CHECK_UNUSED_TSE(W) #endif +static ERTS_INLINE erts_tse_t * +tse_fetch(erts_pix_lock_t *pix_lock) +{ + erts_tse_t *tse = erts_tse_fetch(); + if (!tse->udata) { + erts_proc_lock_queues_t *qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_unlock(pix_lock); +#endif + erts_smp_spin_lock(&qs_lock); + qs = queue_free_list; + if (qs) { + queue_free_list = queue_free_list->next; + erts_smp_spin_unlock(&qs_lock); + } + else { + erts_smp_spin_unlock(&qs_lock); + qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, + sizeof(erts_proc_lock_queues_t)); + sys_memcpy((void *) qs, + (void *) &zeroqs, + sizeof(erts_proc_lock_queues_t)); + } + tse->udata = qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_lock(pix_lock); +#endif + } + tse->uflgs = 0; + return tse; +} static ERTS_INLINE void -free_wtr(erts_proc_lock_waiter_t *wtr) +tse_return(erts_tse_t *tse, int force_free_q) { - CHECK_UNUSED_WAITER(wtr); - erts_smp_spin_lock(&wtr_lock); - wtr->next = waiter_free_list; - waiter_free_list = wtr; - wtr->queues->next = queue_free_list; - queue_free_list = wtr->queues; - erts_smp_spin_unlock(&wtr_lock); + CHECK_UNUSED_TSE(tse); + if (force_free_q || erts_tse_is_tmp(tse)) { + erts_proc_lock_queues_t *qs = tse->udata; + ASSERT(qs); + erts_smp_spin_lock(&qs_lock); + qs->next = queue_free_list; + queue_free_list = qs; + erts_smp_spin_unlock(&qs_lock); + tse->udata = NULL; + } + erts_tse_return(tse); } void erts_proc_lock_prepare_proc_lock_waiter(void) { - erts_tsd_set(waiter_key, (void *) alloc_wtr()); + tse_return(tse_fetch(NULL), 0); } static void -cleanup_waiter(void) +cleanup_tse(void) { - erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key); - if (wtr) - free_wtr(wtr); + erts_tse_t *tse = erts_tse_fetch(); + if (tse) { + if (tse->udata) + tse_return(tse, 1); + else + erts_tse_return(tse); + } } @@ -250,7 +251,7 @@ cleanup_waiter(void) static ERTS_INLINE void enqueue_waiter(erts_proc_lock_queues_t *qs, int ix, - erts_proc_lock_waiter_t *wtr) + erts_tse_t *wtr) { if (!qs->queue[ix]) { qs->queue[ix] = wtr; @@ -266,10 +267,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs, } } -static erts_proc_lock_waiter_t * +static erts_tse_t * dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) { - erts_proc_lock_waiter_t *wtr = qs->queue[ix]; + erts_tse_t *wtr = qs->queue[ix]; ERTS_LC_ASSERT(qs->queue[ix]); if (wtr->next == wtr) { ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr); @@ -295,10 +296,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) * lock. */ static ERTS_INLINE void -try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) +try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; - ErtsProcLocks locks = wtr->wait_locks; + ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); @@ -334,7 +335,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) } } - wtr->wait_locks &= ~got_locks; + wtr->uflgs &= ~got_locks; } /* @@ -350,8 +351,8 @@ transfer_locks(Process *p, int unlock) { int transferred = 0; - erts_proc_lock_waiter_t *wake = NULL; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wake = NULL; + erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; @@ -377,11 +378,11 @@ transfer_locks(Process *p, ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; - ERTS_LC_ASSERT(wtr->wait_locks & lock); - wtr->wait_locks &= ~lock; - if (wtr->wait_locks) + ERTS_LC_ASSERT(wtr->uflgs & lock); + wtr->uflgs &= ~lock; + if (wtr->uflgs) try_aquire(&p->lock, wtr); - if (!wtr->wait_locks) { + if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. @@ -412,9 +413,10 @@ transfer_locks(Process *p, erts_pix_unlock(pix_lock); do { - erts_proc_lock_waiter_t *tmp = wake; + erts_tse_t *tmp = wake; wake = wake->next; - erts_smp_gate_let_through(&tmp->gate, 1); + erts_atomic_set(&tmp->uaflgs, 0); + erts_tse_set(tmp); } while (wake); if (!unlock) @@ -462,26 +464,16 @@ wait_for_locks(Process *p, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); - int tsd; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; + erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ - wtr = erts_tsd_get(waiter_key); - if (wtr) - tsd = 1; - else { -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_unlock(pix_lock); -#endif - wtr = alloc_wtr(); - tsd = 0; -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_lock(pix_lock); -#endif - } + wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ - wtr->wait_locks = need_locks; + wtr->uflgs = need_locks; + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -489,14 +481,16 @@ wait_for_locks(Process *p, ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); + qs = wtr->udata; + ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { - wtr->queues->next = NULL; - p->lock.queues = wtr->queues; + qs->next = NULL; + p->lock.queues = qs; } else { - wtr->queues->next = p->lock.queues->next; - p->lock.queues->next = wtr->queues; + qs->next = p->lock.queues->next; + p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG @@ -506,46 +500,59 @@ wait_for_locks(Process *p, /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif - if (wtr->wait_locks) { /* We didn't get them all; need to wait... */ - /* Got to wait for locks... */ + if (wtr->uflgs) { + /* We didn't get them all; need to wait... */ + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + + erts_atomic_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); - /* - * Wait for needed locks. When we return all needed locks have - * have been acquired by other threads and transfered to us. - */ -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count); -#else - erts_smp_gate_wait(&wtr->gate); -#endif + while (1) { + int res; + erts_tse_reset(wtr); + + if (erts_atomic_read(&wtr->uaflgs) == 0) + break; + + /* + * Wait for needed locks. When we are woken all needed locks have + * have been acquired by other threads and transfered to us. + * However, we need to be prepared for spurious wakeups. + */ + do { + res = erts_tse_wait(wtr); /* might return EINTR */ + } while (res != 0); + } erts_pix_lock(pix_lock); + + ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { - wtr->queues = p->lock.queues->next; - p->lock.queues->next = wtr->queues->next; + qs = p->lock.queues->next; + p->lock.queues->next = qs->next; } else { - wtr->queues = p->lock.queues; + qs = p->lock.queues; p->lock.queues = NULL; } + wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); - if (tsd) - CHECK_UNUSED_WAITER(wtr); - else - free_wtr(wtr); + tse_return(wtr, 0); } /* @@ -563,52 +570,57 @@ erts_proc_lock_failed(Process *p, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - int spin_count = 0; -#else - int spin_count = proc_lock_spin_count; -#endif - + int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + int thr_spin_count; + int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; - while (need_locks != 0) - { - ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks); + if (erts_thr_get_main_status()) + thr_spin_count = proc_lock_spin_count; + else + thr_spin_count = aux_thr_proc_lock_spin_count; + + spin_count = thr_spin_count; + + while (need_locks != 0) { + ErtsProcLocks can_grab; + + can_grab = in_order_locks(olflgs, need_locks); - if (can_grab == 0) - { + if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ - if (spin_count-- <= 0) - { + if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } + ERTS_SPIN_BODY; + + if (--until_yield == 0) { + until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } - else - { + else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); - if (nflgs == olflgs) - { + if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; -#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE /* Since we made progress, reset the spin count. */ - spin_count = proc_lock_spin_count; -#endif + spin_count = thr_spin_count; } - else - { + else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } @@ -1407,7 +1419,7 @@ check_queue(erts_proc_lock_t *lck) wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index d71e5a0a6e..7cfc9893fa 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_t mtx; -#else erts_smp_spinlock_t spnlck; -#endif char buf[64]; /* Try to get locks in different cache lines */ } u; } erts_pix_lock_t; @@ -277,9 +273,12 @@ typedef struct { ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_acqb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_relb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) @@ -289,6 +288,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *, ErtsProcLocks); ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *, ErtsProcLocks); +ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *, + ErtsProcLocks, + ErtsProcLocks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -322,7 +324,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) @@ -348,9 +352,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, - erts_pix_lock_t *, - ErtsProcLocks, - char *file, unsigned int line); + erts_pix_lock_t *, + ErtsProcLocks, + char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, erts_pix_lock_t *, @@ -372,30 +376,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_lock(&pixlck->u.mtx); -#else erts_smp_spin_lock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_unlock(&pixlck->u.mtx); -#else erts_smp_spin_unlock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { -#if ERTS_PROC_LOCK_MUTEX_IMPL - return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx); -#else return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); -#endif } /* @@ -417,9 +409,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) ErtsProcLocks expct_lflgs = 0; while (1) { - ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, - expct_lflgs | locks, - expct_lflgs); + ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, + expct_lflgs | locks, + expct_lflgs); if (ERTS_LIKELY(lflgs == expct_lflgs)) { /* We successfully grabbed all locks. */ return 0; @@ -535,7 +527,7 @@ erts_smp_proc_unlock__(Process *p, if (want_lflgs != old_lflgs) { ErtsProcLocks new_lflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs); if (new_lflgs != old_lflgs) { /* cmpxchg failed, try again. */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 03d2a586e3..b41fa70476 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -43,9 +43,17 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t; typedef erts_tid_t erts_smp_tid_t; typedef erts_mtx_t erts_smp_mtx_t; typedef erts_cnd_t erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER +#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED +#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED +typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_gate_t erts_smp_gate_t; typedef ethr_atomic_t erts_smp_atomic_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; @@ -54,15 +62,27 @@ void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0 +#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} typedef int erts_smp_thr_opts_t; typedef int erts_smp_thr_init_data_t; typedef int erts_smp_tid_t; typedef int erts_smp_mtx_t; typedef int erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_LONG_LIVED 0 +#define ERTS_SMP_RWMTX_SHORT_LIVED 0 +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef int erts_smp_gate_t; typedef long erts_smp_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; @@ -103,8 +123,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); @@ -119,9 +137,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); +ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); @@ -155,6 +181,16 @@ ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -190,12 +226,6 @@ ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount); #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 @@ -331,22 +361,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_set_forksafe(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unset_forksafe(mtx); -#endif -} - ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) { @@ -433,6 +447,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) } ERTS_GLB_INLINE void +erts_smp_rwmtx_set_reader_group(int no) +{ +#ifdef ERTS_SMP + erts_rwmtx_set_reader_group(no); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -441,6 +474,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt(rwmtx, opt, name); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) { #ifdef ERTS_SMP @@ -709,6 +752,73 @@ erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) #endif } +ERTS_GLB_INLINE long +erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i) +{ +#ifdef ERTS_SMP + erts_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { @@ -919,54 +1029,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_smp_gate_init(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_init((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_destroy(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_destroy((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_close(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_close((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no) -{ -#ifdef ERTS_SMP - erts_gate_let_through((erts_gate_t *) gp, no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_wait(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_wait((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount) -{ -#ifdef ERTS_SMP - erts_gate_swait((erts_gate_t *) gp, spincount); -#endif -} - #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 35b338c6eb..0b7269262e 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -25,6 +25,11 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +#define ERTS_SPIN_BODY ETHR_SPIN_BODY + +#define ERTS_MAX_READER_GROUPS 8 +extern int erts_reader_groups; + #include "sys.h" #ifdef USE_THREADS @@ -48,6 +53,7 @@ #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; +typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ @@ -73,8 +79,19 @@ typedef struct { erts_lcnt_lock_t lcnt; #endif } erts_rwmtx_t; + +#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER +#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL +#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED +#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED +#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED +typedef ethr_rwmutex_opt erts_rwmtx_opt_t; + typedef ethr_tsd_key erts_tsd_key_t; -typedef ethr_gate erts_gate_t; +typedef ethr_ts_event erts_tse_t; typedef ethr_atomic_t erts_atomic_t; /* spinlock */ @@ -103,25 +120,14 @@ typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_REC_MTX_INITER \ - {ETHR_REC_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)} -#define ERTS_MTX_INITER \ - {ETHR_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)} -#else -#define ERTS_REC_MTX_INITER {ETHR_REC_MUTEX_INITER} -#define ERTS_MTX_INITER {ETHR_MUTEX_INITER} -#endif -#define ERTS_CND_INITER ETHR_COND_INITER #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER +#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ + ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif - #else /* #ifdef USE_THREADS */ #define ERTS_THR_MEMORY_BARRIER @@ -129,12 +135,26 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; typedef int erts_thr_init_data_t; +typedef int erts_thr_late_init_data_t; typedef int erts_tid_t; typedef int erts_mtx_t; typedef int erts_cnd_t; +#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_RWMTX_TYPE_NORMAL 0 +#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_RWMTX_LONG_LIVED 0 +#define ERTS_RWMTX_SHORT_LIVED 0 +#define ERTS_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_rwmtx_opt_t; typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; -typedef int erts_gate_t; +typedef int erts_tse_t; typedef long erts_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -148,7 +168,6 @@ typedef struct { long tv_nsec; } erts_thr_timeval_t; -#define ERTS_REC_MTX_INITER 0 #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -158,6 +177,7 @@ typedef struct { #endif /* #ifdef USE_THREADS */ ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); +ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); @@ -166,9 +186,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx); -#endif ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, @@ -177,8 +194,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); @@ -192,9 +207,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); +ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); @@ -228,6 +251,20 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, + char *name, + Eterm extra, + Uint16 opt); ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra); @@ -263,12 +300,16 @@ ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); -ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount); +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); +ERTS_GLB_INLINE int erts_thr_get_main_status(void); +ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 @@ -290,6 +331,16 @@ erts_thr_init(erts_thr_init_data_t *id) } ERTS_GLB_INLINE void +erts_thr_late_init(erts_thr_late_init_data_t *id) +{ +#ifdef USE_THREADS + int res = ethr_late_init(id); + if (res) + erts_thr_fatal_error(res, "complete initialization of thread library"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { @@ -362,20 +413,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) #endif } - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_rec_mtx_init(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_rec_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize recursive mutex"); -#endif -} -#endif - - ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { @@ -422,9 +459,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -463,9 +498,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -492,26 +525,6 @@ erts_mtx_destroy(erts_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_mtx_set_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_set_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "set mutex forksafe"); -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_unset_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_unset_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "unset mutex forksafe"); -#endif -} - ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx) { @@ -531,11 +544,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); -#endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try lock mutex"); - +#endif return res; #else return 0; @@ -551,19 +560,16 @@ erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif - res = ethr_mutex_lock(&mtx->mtx); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "lock mutex"); #endif } @@ -571,16 +577,13 @@ ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif - res = ethr_mutex_unlock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "unlock mutex"); + ethr_mutex_unlock(&mtx->mtx); #endif } @@ -648,9 +651,7 @@ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_signal(cnd); - if (res) - erts_thr_fatal_error(res, "signal on condition variable"); + ethr_cond_signal(cnd); #endif } @@ -659,19 +660,34 @@ ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_broadcast(cnd); - if (res) - erts_thr_fatal_error(res, "broadcast on condition variable"); + ethr_cond_broadcast(cnd); #endif } /* rwmutex */ ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) +erts_rwmtx_set_reader_group(int no) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); +#endif + res = ethr_rwmutex_set_reader_group(no); + if (res != 0) + erts_thr_fatal_error(res, "set reader group"); +#endif +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef USE_THREADS + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -684,10 +700,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, + char *name, + Eterm extra) +{ + erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -700,6 +726,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) } ERTS_GLB_INLINE void +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +{ + erts_rwmtx_init_opt(rwmtx, NULL, name); +} + +ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS @@ -736,9 +768,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try read lock rwmutex"); return res; #else @@ -754,19 +783,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_rlock(&rwmtx->rwmtx); + ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "read lock rwmutex"); #endif } @@ -774,16 +800,13 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_runlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "read unlock rwmutex"); + ethr_rwmutex_runlock(&rwmtx->rwmtx); #endif } @@ -808,9 +831,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try write lock rwmutex"); return res; #else @@ -826,19 +846,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwlock(&rwmtx->rwmtx); + ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "write lock rwmutex"); #endif } @@ -846,16 +863,13 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "write unlock rwmutex"); + ethr_rwmutex_rwunlock(&rwmtx->rwmtx); #endif } @@ -917,9 +931,7 @@ ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_init(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic init"); + ethr_atomic_init(var, i); #else *var = i; #endif @@ -929,9 +941,7 @@ ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_set(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic set"); + ethr_atomic_set(var, i); #else *var = i; #endif @@ -941,11 +951,7 @@ ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS - long i; - int res = ethr_atomic_read(var, &i); - if (res) - erts_thr_fatal_error(res, "perform atomic read"); - return i; + return ethr_atomic_read(var); #else return *var; #endif @@ -955,11 +961,7 @@ ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_inctest(incp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic increment and test"); - return test; + return ethr_atomic_inc_read(incp); #else return ++(*incp); #endif @@ -969,11 +971,7 @@ ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_dectest(decp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement and test"); - return test; + return ethr_atomic_dec_read(decp); #else return --(*decp); #endif @@ -983,9 +981,7 @@ ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp) { #ifdef USE_THREADS - int res = ethr_atomic_inc(incp); - if (res) - erts_thr_fatal_error(res, "perform atomic increment"); + ethr_atomic_inc(incp); #else ++(*incp); #endif @@ -995,9 +991,7 @@ ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp) { #ifdef USE_THREADS - int res = ethr_atomic_dec(decp); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement"); + ethr_atomic_dec(decp); #else --(*decp); #endif @@ -1007,11 +1001,7 @@ ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_addtest(addp, i, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic addition and test"); - return test; + return ethr_atomic_add_read(addp, i); #else return *addp += i; #endif @@ -1021,9 +1011,7 @@ ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - int res = ethr_atomic_add(addp, i); - if (res) - erts_thr_fatal_error(res, "perform atomic addition"); + ethr_atomic_add(addp, i); #else *addp += i; #endif @@ -1034,9 +1022,7 @@ erts_atomic_xchg(erts_atomic_t *xchgp, long new) { long old; #ifdef USE_THREADS - int res = ethr_atomic_xchg(xchgp, new, &old); - if (res) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_xchg(xchgp, new); #else old = *xchgp; *xchgp = new; @@ -1048,11 +1034,7 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) { #ifdef USE_THREADS - long old; - int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old); - if (ERTS_UNLIKELY(res != 0)) - erts_thr_fatal_error(res, "perform atomic exchange"); - return old; + return ethr_atomic_cmpxchg(xchgp, new, expected); #else long old = *xchgp; if (old == expected) @@ -1064,31 +1046,95 @@ erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_or_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise or"); + return ethr_atomic_read_bor(var, mask); #else + long old; old = *var; *var |= mask; -#endif return old; +#endif } ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_and_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise and"); + return ethr_atomic_read_band(var, mask); #else + long old; old = *var; *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE long +erts_atomic_read_acqb(erts_atomic_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_set_relb(erts_atomic_t *var, long i) +{ +#ifdef USE_THREADS + ethr_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_dec_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_atomic_dectest_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic_dec_read_relb(decp); +#else + return --(*decp); #endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif } /* spinlock */ @@ -1112,6 +1158,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, + Uint16 opt) +{ +#ifdef USE_THREADS + int res = ethr_spinlock_init(&lock->slck); + if (res) + erts_thr_fatal_error(res, "init spinlock"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); +#endif +#else + (void)lock; +#endif +} + + +ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name) { #ifdef USE_THREADS @@ -1152,16 +1218,13 @@ ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif - res = ethr_spin_unlock(&lock->slck); - if (res) - erts_thr_fatal_error(res, "release spin lock"); + ethr_spin_unlock(&lock->slck); #else (void)lock; #endif @@ -1175,19 +1238,16 @@ erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif - res = ethr_spin_lock(&lock->slck); + ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take spin lock"); #else (void)lock; #endif @@ -1268,16 +1328,13 @@ ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release read lock"); + ethr_read_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1291,19 +1348,16 @@ erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_lock(&lock->rwlck); + ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take read lock"); #else (void)lock; #endif @@ -1313,16 +1367,13 @@ ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release write lock"); + ethr_write_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1336,19 +1387,16 @@ erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_lock(&lock->rwlck); + ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take write lock"); #else (void)lock; #endif @@ -1432,66 +1480,95 @@ erts_tsd_get(erts_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_gate_init(erts_gate_t *gp) +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { #ifdef USE_THREADS - int res = ethr_gate_init((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "initialize gate"); + return (erts_tse_t *) ethr_get_ts_event(); +#else + return (erts_tse_t *) NULL; #endif } -ERTS_GLB_INLINE void -erts_gate_destroy(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_destroy((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "destroy gate"); + ethr_leave_ts_event(ep); #endif } -ERTS_GLB_INLINE void -erts_gate_close(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_close((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "close gate"); + ethr_event_set(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_let_through(erts_gate_t *gp, unsigned no) +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_let_through((ethr_gate *) gp, no); - if (res != 0) - erts_thr_fatal_error(res, "let through gate"); + ethr_event_reset(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_wait(erts_gate_t *gp) +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return ethr_event_wait(&((ethr_ts_event *) ep)->event); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) +{ +#ifdef USE_THREADS + return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_wait((ethr_gate *) gp); + return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) +{ +#ifdef USE_THREADS + int res = ethr_set_main_thr_status(on, no); if (res != 0) - erts_thr_fatal_error(res, "wait on gate"); + erts_thr_fatal_error(res, "set thread main status"); #endif } -ERTS_GLB_INLINE void -erts_gate_swait(erts_gate_t *gp, int spincount) +ERTS_GLB_INLINE int erts_thr_get_main_status(void) { #ifdef USE_THREADS - int res = ethr_gate_swait((ethr_gate *) gp, spincount); + int main_status; + int res = ethr_get_main_thr_status(&main_status); if (res != 0) - erts_thr_fatal_error(res, "swait on gate"); + erts_thr_fatal_error(res, "get thread main status"); + return main_status; +#else + return 1; #endif } +ERTS_GLB_INLINE void erts_thr_yield(void) +{ +#ifdef USE_THREADS + int res = ETHR_YIELD(); + if (res != 0) + erts_thr_fatal_error(res, "yield"); +#endif +} + + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 66b05c0e9d..5e81a2d624 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -43,8 +43,6 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. #define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) #define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) #define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) -#define export_init_lock() erts_smp_rwmtx_init(&export_table_lock, \ - "export_tab") extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; @@ -111,8 +109,12 @@ void init_export_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); - export_init_lock(); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 900ebcbbf7..c9bb7bbe91 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -39,8 +39,6 @@ static Hash process_reg; static erts_smp_rwmtx_t regtab_rwmtx; -#define reg_lock_init() erts_smp_rwmtx_init(®tab_rwmtx, \ - "reg_tab") #define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) #define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) #define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) @@ -147,8 +145,11 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - reg_lock_init(); + erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 737ffd9f94..400ef6c0ce 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -384,18 +384,6 @@ MALLOC_USE_HASH(1); #endif #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} #ifdef ERTS_THR_HAVE_SIG_FUNCS /* @@ -488,9 +476,6 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -538,13 +523,14 @@ erts_sys_pre_init(void) #endif #endif /* USE_THREADS */ erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); } void erl_sys_init(void) { + erts_smp_rwmtx_init(&environ_rwmtx, "environ"); #if !DISABLE_VFORK + { int res; char bindir[MAXPATHLEN]; size_t bindirsz = sizeof(bindir); @@ -574,6 +560,7 @@ erl_sys_init(void) bindir, DIR_SEPARATOR_CHAR, CHILD_SETUP_PROG_NAME); + } #endif #ifdef USE_SETLINEBUF diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index bd02cf85eb..54f71b202d 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -2974,19 +2974,6 @@ check_supported_os_version(void) } #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} - #ifdef ERTS_ENABLE_LOCK_COUNT static void thr_create_prepare_child(void *vtcdp) @@ -3005,14 +2992,9 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; - #ifdef ERTS_ENABLE_LOCK_COUNT eid.thread_create_child_func = thr_create_prepare_child; #endif - erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init(); diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index ab727b7cb1..4f4458802c 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -33,7 +33,7 @@ -include("test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). -export([equal/1, few_low/1, @@ -49,7 +49,8 @@ cpu_topology/1, sct_cmd/1, sbt_cmd/1, - scheduler_suspend/1]). + scheduler_suspend/1, + reader_groups/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(10)). @@ -67,7 +68,8 @@ all(suite) -> equal_with_high_max, bound_process, scheduler_bind, - scheduler_suspend]. + scheduler_suspend, + reader_groups]. init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), @@ -81,6 +83,10 @@ fin_per_testcase(_Case, Config) when is_list(Config) -> ?t:timetrap_cancel(Dog), ok. +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)). -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000). @@ -965,12 +971,361 @@ sst3_loop(S, N) -> erlang:system_flag(schedulers_online, 1), erlang:system_flag(schedulers_online, S), sst3_loop(S, N-1). + +reader_groups(Config) when is_list(Config) -> + %% White box testing. These results are correct, but other results + %% could be too... + + %% The actual tilepro64 topology + CPUT0 = [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}], + + ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1}, + {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2}, + {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4}, + {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5}, + {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5}, + {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6}, + {58,8},{60,6},{61,6}] + = reader_groups_map(CPUT0, 8), + + CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]), + c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]), + c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]), + c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]), + p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]), + c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]), + c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]), + c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]), + n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]), + c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]), + c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]), + c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]), + p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]), + c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]), + c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]), + c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]), + n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]), + c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]), + c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]), + c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]), + p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]), + c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]), + c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]), + c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]), + n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]), + c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]), + c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]), + c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]), + p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]), + c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]), + c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]), + c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3}, + {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5}, + {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7}, + {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10}, + {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12}, + {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14}, + {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16}, + {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18}, + {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20}, + {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22}, + {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24}, + {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26}, + {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27}, + {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29}, + {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31}, + {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}] + = reader_groups_map(CPUT1, 128), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1}, + {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1}, + {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1}, + {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1}, + {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1}, + {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2}, + {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2}, + {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2}, + {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2}, + {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2}, + {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2}, + {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2}, + {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2}, + {125,2},{126,2},{127,2}] + = reader_groups_map(CPUT1, 2), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3}, + {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4}, + {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5}, + {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6}, + {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7}, + {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8}, + {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10}, + {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11}, + {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12}, + {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13}, + {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14}, + {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14}, + {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15}, + {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16}, + {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17}, + {125,17},{126,17},{127,17}] + = reader_groups_map(CPUT1, 17), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2}, + {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2}, + {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3}, + {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3}, + {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4}, + {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5}, + {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5}, + {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6}, + {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6}, + {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7}, + {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7}, + {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7}, + {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7}, + {125,7},{126,7},{127,7}] + = reader_groups_map(CPUT1, 7), + + ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]), + p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2},{6,2},{7,2},{8,2},{9,2}, + {10,3}, + {11,4},{12,4},{13,4}, + {14,5},{15,5}] = reader_groups_map(CPUT2, 5), + + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,5},{13,5}, + {14,6},{15,6}] = reader_groups_map(CPUT2, 6), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,7}] = reader_groups_map(CPUT2, 7), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,8}] = reader_groups_map(CPUT2, 8), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,7}, + {14,8},{15,9}] = reader_groups_map(CPUT2, 9), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,8}, + {14,9},{15,10}] = reader_groups_map(CPUT2, 10), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,4}, + {5,5},{6,5},{7,5},{8,5},{9,5}, + {10,6}, + {11,7},{12,8},{13,9}, + {14,10},{15,11}] = reader_groups_map(CPUT2, 11), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,5}, + {5,6},{6,6},{7,6},{8,6},{9,6}, + {10,7}, + {11,8},{12,9},{13,10}, + {14,11},{15,12}] = reader_groups_map(CPUT2, 100), + + CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))]), + p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])], + + ?line [{0,5},{1,5},{2,6},{3,6},{4,6}, + {5,1},{6,1},{7,1},{8,1},{9,1}, + {10,2},{11,3},{12,3},{13,3}, + {14,4},{15,4}] = reader_groups_map(CPUT3, 6), + + CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]), + p([t(l(5))]), + p([c(l(6)),c(l(7)),c(l(8))]), + p([c(l(9)),c(l(10))]), + p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,3},{8,3}, + {9,4},{10,4}, + {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,4},{8,4}, + {9,5},{10,5}, + {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7), + + ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10), + + ?line ok. +reader_groups_map(CPUT, Groups) -> + Old = erlang:system_info({cpu_topology, defined}), + erlang:system_flag(cpu_topology, CPUT), + enable_internal_state(), + Res = erts_debug:get_internal_state({reader_groups_map, Groups}), + erlang:system_flag(cpu_topology, Old), + lists:sort(Res). + %% %% Utils %% +tilera_cpu_topology() -> + [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}]. + +l(Id) -> + {logical, Id}. + +t(X) -> + {thread, X}. + +c(X) -> + {core, X}. + +p(X) -> + {processor, X}. + +n(X) -> + {node, X}. + mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index f79f5cc978..76ce0a7e3c 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -131,6 +131,12 @@ static char *plush_val_switches[] = { NULL }; +/* +r arguments with values */ +static char *plusr_val_switches[] = { + "g", + NULL +}; + /* * Define sleep(seconds) in terms of Sleep() on Windows. @@ -872,6 +878,21 @@ int main(int argc, char **argv) i++; } break; + case 'r': + if (!is_one_of_strings(&argv[i][2], + plusr_val_switches)) + goto the_default; + else { + if (i+1 >= argc + || argv[i+1][0] == '-' + || argv[i+1][0] == '+') + usage(argv[i]); + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + } + break; case 's': if (!is_one_of_strings(&argv[i][2], pluss_val_switches)) @@ -1069,11 +1090,12 @@ usage_aux(void) "[-hybrid] " #endif "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] " - "[-args_file FILENAME] " - "[+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] " + "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] " + "[+h HEAP_SIZE_OPTION] [+K BOOLEAN] " "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+R COMPAT_REL] " - "[+r] [+s SCHEDULER_OPTION] [+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] [+W<i|w>] " - "[args ...]\n"); + "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] " + "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] " + "[+W<i|w>] [args ...]\n"); exit(1); } diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h index 82e9ba3798..6b875ff824 100644 --- a/erts/include/internal/erl_misc_utils.h +++ b/erts/include/internal/erl_misc_utils.h @@ -50,4 +50,8 @@ int erts_unbind_from_cpu_str(char *str); int erts_milli_sleep(long); +#ifdef __WIN32__ +int erts_get_last_win_errno(void); +#endif + #endif /* #ifndef ERL_MISC_UTILS_H_ */ diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h new file mode 100644 index 0000000000..e9c3daf783 --- /dev/null +++ b/erts/include/internal/ethr_internal.h @@ -0,0 +1,67 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Internal ethread exports + * Author: Rickard Green + */ + +#ifndef ETHR_INTERNAL_H__ +#define ETHR_INTERNAL_H__ + +#include "erl_misc_utils.h" + +extern ethr_memory_allocators ethr_mem__; +extern erts_cpu_info_t *ethr_cpu_info__; +extern size_t ethr_pagesize__; +extern size_t ethr_min_stack_size__; /* kilo words */ +extern size_t ethr_max_stack_size__; /* kilo words */ +extern int ethr_not_completely_inited__; +extern int ethr_not_inited__; + +extern void *(*ethr_thr_prepare_func__)(void); +extern void (*ethr_thr_parent_func__)(void *); +extern void (*ethr_thr_child_func__)(void *); + +#define ETHR_PAGE_ALIGN(SZ) \ + (((((size_t) (SZ)) - 1)/ethr_pagesize__ + 1)*ethr_pagesize__) +#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) +#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) + +#undef ETHR_STACK_GUARD_SIZE +#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE +# define ETHR_STACK_GUARD_SIZE (ethr_pagesize__) +#endif + +/* implemented in lib_src/<thr-lib>/ethread.c */ +int ethr_set_tse__(ethr_ts_event *tsep); +ethr_ts_event *ethr_get_tse__(void); +ETHR_PROTO_NORETURN__ ethr_abort__(void); +#ifdef ETHR_WIN32_THREADS +int ethr_win_get_errno__(void); +#endif + +/* implemented in lib_src/common/ethread_aux.c */ +int ethr_init_common__(ethr_init_data *id); +int ethr_late_init_common__(ethr_late_init_data *lid); +void ethr_run_exit_handlers__(void); +void ethr_ts_event_destructor__(void *vtsep); + + +#endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h new file mode 100644 index 0000000000..4ce3e75c78 --- /dev/null +++ b/erts/include/internal/ethr_mutex.h @@ -0,0 +1,510 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifndef ETHR_MUTEX_H__ +#define ETHR_MUTEX_H__ + +#define ETHR_RWMUTEX_INITIALIZED 0x99999999 +#define ETHR_MUTEX_INITIALIZED 0x77777777 +#define ETHR_COND_INITIALIZED 0x55555555 + +#if 0 +# define ETHR_MTX_HARD_DEBUG +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +# ifdef __GNUC__ +# warning ETHR_MTX_HARD_DEBUG +# endif +/*# define ETHR_MTX_HARD_DEBUG_LFS*/ +/*# define ETHR_MTX_HARD_DEBUG_FENCE*/ +/*# define ETHR_MTX_HARD_DEBUG_Q*/ +# define ETHR_MTX_HARD_DEBUG_WSQ + +# if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q) +# define ETHR_MTX_HARD_DEBUG_WSQ +# endif +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#if 0 +# define ETHR_MTX_Q_LOCK_SPINLOCK__ +# define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t +#elif defined(ETHR_PTHREADS) +# define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__ +# define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t +#elif defined(ETHR_WIN32_THREADS) +# define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__ +# define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION +#else +# error Need a qlock implementation +#endif + +#define ETHR_RWMTX_W_FLG__ (((long) 1) << 31) +#define ETHR_RWMTX_W_WAIT_FLG__ (((long) 1) << 30) +#define ETHR_RWMTX_R_WAIT_FLG__ (((long) 1) << 29) + +/* frequent read kind */ +#define ETHR_RWMTX_R_FLG__ (((long) 1) << 28) +#define ETHR_RWMTX_R_PEND_UNLCK_MASK__ (ETHR_RWMTX_R_FLG__ - 1) +#define ETHR_RWMTX_R_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) + +/* normal kind */ +#define ETHR_RWMTX_RS_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) + +#define ETHR_RWMTX_WAIT_FLGS__ \ + (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__) + +#define ETHR_CND_WAIT_FLG__ ETHR_RWMTX_R_WAIT_FLG__ + +struct ethr_mutex_base_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long pre_fence; +#endif + ethr_atomic_t flgs; + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; + short aux_scnt; + short main_scnt; +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + int ws; +#endif +#ifdef ETHR_MTX_HARD_DEBUG_LFS + ethr_atomic_t hdbg_lfs; +#endif +}; + +#endif + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_mutex_opt; + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_cond_opt; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + struct ethr_mutex_base_ mtxb; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + struct { + long pre_fence; + } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */ +#endif + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; + short aux_scnt; + short main_scnt; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread */ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + pthread_mutex_t pt_mtx; +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { + pthread_cond_t pt_cnd; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread */ + +int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *); +int ethr_mutex_init(ethr_mutex *); +int ethr_mutex_destroy(ethr_mutex *); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) +int ethr_mutex_trylock(ethr_mutex *); +void ethr_mutex_lock(ethr_mutex *); +void ethr_mutex_unlock(ethr_mutex *); +#endif +int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *); +int ethr_cond_init(ethr_cond *); +int ethr_cond_destroy(ethr_cond *); +void ethr_cond_signal(ethr_cond *); +void ethr_cond_broadcast(ethr_cond *); +int ethr_cond_wait(ethr_cond *, ethr_mutex *); + +typedef enum { + ETHR_RWMUTEX_TYPE_NORMAL, + ETHR_RWMUTEX_TYPE_FREQUENT_READ, + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +} ethr_rwmutex_type; + +typedef enum { + ETHR_RWMUTEX_LONG_LIVED, + ETHR_RWMUTEX_SHORT_LIVED, + ETHR_RWMUTEX_UNKNOWN_LIVED +} ethr_rwmutex_lived; + +typedef struct { + ethr_rwmutex_type type; + ethr_rwmutex_lived lived; + int main_spincount; + int aux_spincount; +} ethr_rwmutex_opt; + +#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \ + {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +typedef union { + struct { + ethr_atomic_t readers; + int waiting_readers; + int byte_offset; + ethr_rwmutex_lived lived; + } data; + char align__[ETHR_CACHE_LINE_SIZE]; +} ethr_rwmtx_readers_array__; + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + struct ethr_mutex_base_ mtxb; + ethr_rwmutex_type type; + ethr_ts_event *rq_end; + union { + ethr_rwmtx_readers_array__ *ra; + int rs; + } tdata; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread_rwlock */ + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + pthread_rwlock_t pt_rwlock; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread_rwlock */ + +int ethr_rwmutex_set_reader_group(int); +int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *); +int ethr_rwmutex_init(ethr_rwmutex *); +int ethr_rwmutex_destroy(ethr_rwmutex *); +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \ + || !defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_MUTEX_IMPL__) +int ethr_rwmutex_tryrlock(ethr_rwmutex *); +void ethr_rwmutex_rlock(ethr_rwmutex *); +void ethr_rwmutex_runlock(ethr_rwmutex *); +int ethr_rwmutex_tryrwlock(ethr_rwmutex *); +void ethr_rwmutex_rwlock(ethr_rwmutex *); +void ethr_rwmutex_rwunlock(ethr_rwmutex *); +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +#define ETHR_MTX_HARD_ASSERT(A) \ + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) +#else +#define ETHR_MTX_HARD_ASSERT(A) ((void) 1) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_LFS +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) \ +do { \ + ethr_atomic_init(&(MTXB)->hdbg_lfs, 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) \ +do { \ + long val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic_inc_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ > 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) \ +do { \ + long val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ >= 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) \ +do { \ + long val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == -1); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) \ +do { \ + long val__ = ethr_atomic_inctest(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#else +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeadbeefdeadbeefL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeaddeaddeadL +#else +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeaddeadL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeadL +#endif + +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\ + ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \ +do { \ + (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \ + (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \ +} while (0) +#else +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) +#endif + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50 + +#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0 +#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0 + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +void ethr_mutex_lock_wait__(ethr_mutex *, long); +void ethr_mutex_unlock_wake__(ethr_mutex *, long); + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + long act; + int res; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + res = (act == 0) ? 0 : EBUSY; + + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + ETHR_COMPILER_BARRIER; + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + long act; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + ethr_mutex_lock_wait__(mtx, act); + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + long act; + ETHR_COMPILER_BARRIER; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb); + + act = ethr_atomic_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + ethr_mutex_unlock_wake__(mtx, act); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#else /* pthread_mutex */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + int res; + res = pthread_mutex_trylock(&mtx->pt_mtx); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_lock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_unlock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_mutex */ + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50 + +#else /* pthread_rwlock */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_rwlock */ + +int ethr_mutex_lib_init(int); +int ethr_mutex_lib_late_init(int, int); + +#endif /* #ifndef ETHR_MUTEX_H__ */ diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h new file mode 100644 index 0000000000..2f9f987d0b --- /dev/null +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -0,0 +1,185 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: "Optimized" fallbacks used when native ops are missing + * Author: Rickard Green + */ + +#ifndef ETHR_OPTIMIZED_FALLBACKS_H__ +#define ETHR_OPTIMIZED_FALLBACKS_H__ + +#ifdef ETHR_HAVE_NATIVE_ATOMICS +#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 +#endif + +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 +#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) +/* --- Optimized spinlocks using pthread spinlocks -------------------------- */ +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +typedef pthread_spinlock_t ethr_opt_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_init((pthread_spinlock_t *) lock, 0); +} + +static ETHR_INLINE int +ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_destroy((pthread_spinlock_t *) lock); +} + + +static ETHR_INLINE int +ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_unlock((pthread_spinlock_t *) lock); +} + +static ETHR_INLINE int +ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_lock((pthread_spinlock_t *) lock); +} + +#endif + +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native spinlocks using native atomics -------------------------------- */ +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +typedef ethr_native_atomic_t ethr_native_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *lock) +{ + ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) == 1); + ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *lock) +{ + while (ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + (long) 1, (long) 0) != 0) { + ETHR_SPIN_BODY; + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#endif + + +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native rwspinlocks using native atomics ------------------------------ */ +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 + +typedef ethr_native_atomic_t ethr_native_rwlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +#define ETHR_WLOCK_FLAG__ (((long) 1) << 30) + +static ETHR_INLINE void +ethr_native_rwlock_init(ethr_native_rwlock_t *lock) +{ + ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_read_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; +#ifdef DEBUG + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) >= 0); +#endif + ethr_native_atomic_dec_relb((ethr_native_atomic_t *) lock); +} + +static ETHR_INLINE void +ethr_native_read_lock(ethr_native_rwlock_t *lock) +{ + long act, exp = 0; + while (1) { + act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + exp+1, exp); + if (act == exp) + break; + ETHR_SPIN_BODY; + exp = (act & ETHR_WLOCK_FLAG__) ? 0 : act; + } + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ethr_native_write_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) + == ETHR_WLOCK_FLAG__); + ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_write_lock(ethr_native_rwlock_t *lock) +{ + long act, exp = 0; + while (1) { + act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + exp|ETHR_WLOCK_FLAG__, exp); + if (act == exp) + break; + ETHR_SPIN_BODY; + exp = act & ~ETHR_WLOCK_FLAG__; + } + act |= ETHR_WLOCK_FLAG__; + /* Wait for readers to leave */ + while (act != ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ethr_native_atomic_read_acqb((ethr_native_atomic_t *) lock); + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#endif + +#endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 4e7a38cd5c..4a205699bd 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -33,20 +33,9 @@ #include <stdlib.h> #include "erl_errno.h" -/* - * Extra memory barrier requirements: - * - ethr_atomic_or_old() needs to enforce a memory barrier sufficient - * for a lock operation. - * - ethr_atomic_and_old() needs to enforce a memory barrier sufficient - * for an unlock operation. - * - ethr_atomic_cmpxchg() needs to enforce a memory barrier sufficient - * for a lock and unlock operation. - */ - - -#undef ETHR_USE_RWMTX_FALLBACK #undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#undef ETHR_HAVE_OPTIMIZED_LOCKS +#undef ETHR_HAVE_OPTIMIZED_SPINLOCK +#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK typedef struct { long tv_sec; @@ -54,6 +43,10 @@ typedef struct { } ethr_timeval; #if defined(DEBUG) +# define ETHR_DEBUG +#endif + +#if defined(ETHR_DEBUG) # undef ETHR_XCHK # define ETHR_XCHK 1 #else @@ -68,47 +61,57 @@ typedef struct { #elif defined(__WIN32__) # define ETHR_INLINE __forceinline #endif -#if defined(DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ +#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC)) # undef ETHR_INLINE # define ETHR_INLINE # undef ETHR_TRY_INLINE_FUNCS #endif -#ifdef ETHR_FORCE_INLINE_FUNCS -# define ETHR_TRY_INLINE_FUNCS -#endif -#if !defined(ETHR_DISABLE_NATIVE_IMPLS) \ - && (defined(PURIFY) || defined(VALGRIND) || defined(ERTS_MIXED_CYGWIN_VC)) +#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && (defined(PURIFY)||defined(VALGRIND)) # define ETHR_DISABLE_NATIVE_IMPLS #endif -#define ETHR_RWMUTEX_INITIALIZED 0x99999999 -#define ETHR_MUTEX_INITIALIZED 0x77777777 -#define ETHR_COND_INITIALIZED 0x55555555 +/* Assume 64-byte cache line size */ +#define ETHR_CACHE_LINE_SIZE 64L +#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) -#define ETHR_CACHE_LINE_SIZE 64 +#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ + (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE) -#ifdef ETHR_INLINE_FUNC_NAME_ -# define ETHR_CUSTOM_INLINE_FUNC_NAME_ -#else +#ifndef ETHR_INLINE_FUNC_NAME_ # define ETHR_INLINE_FUNC_NAME_(X) X #endif -#define ETHR_COMPILER_BARRIER ethr_compiler_barrier() -#ifdef __GNUC__ -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER __asm__ __volatile__("":::"memory") +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif #endif -#ifdef DEBUG +int ethr_assert_failed(const char *file, int line, const char *func, char *a); +#ifdef ETHR_DEBUG #define ETHR_ASSERT(A) \ - ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) -int ethr_assert_failed(char *f, int l, char *a); + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A))) #else #define ETHR_ASSERT(A) ((void) 1) #endif +#if defined(__GNUC__) +# define ETHR_PROTO_NORETURN__ void __attribute__((noreturn)) +# define ETHR_IMPL_NORETURN__ void +#elif defined(__WIN32__) && defined(_MSC_VER) +# define ETHR_PROTO_NORETURN__ __declspec(noreturn) void +# define ETHR_IMPL_NORETURN__ __declspec(noreturn) void +#else +# define ETHR_PROTO_NORETURN__ void +# define ETHR_IMPL_NORETURN__ void +#endif + #if defined(ETHR_PTHREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The pthread implementation * @@ -118,7 +121,9 @@ int ethr_assert_failed(char *f, int l, char *a); #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE." #endif -#if defined(ETHR_HAVE_MIT_PTHREAD_H) +#if defined(ETHR_NEED_NPTL_PTHREAD_H) +#include <nptl/pthread.h> +#elif defined(ETHR_HAVE_MIT_PTHREAD_H) #include <pthread/mit/pthread.h> #elif defined(ETHR_HAVE_PTHREAD_H) #include <pthread.h> @@ -128,130 +133,23 @@ int ethr_assert_failed(char *f, int l, char *a); typedef pthread_t ethr_tid; -typedef struct ethr_mutex_ ethr_mutex; -struct ethr_mutex_ { - pthread_mutex_t pt_mtx; - int is_rec_mtx; - ethr_mutex *prev; - ethr_mutex *next; -#if ETHR_XCHK - int initialized; -#endif -}; - -typedef struct ethr_cond_ ethr_cond; -struct ethr_cond_ { - pthread_cond_t pt_cnd; -#if ETHR_XCHK - int initialized; -#endif -}; +typedef pthread_key_t ethr_tsd_key; -#ifndef ETHR_HAVE_PTHREAD_RWLOCK_INIT -#define ETHR_USE_RWMTX_FALLBACK -#else -typedef struct ethr_rwmutex_ ethr_rwmutex; -struct ethr_rwmutex_ { - pthread_rwlock_t pt_rwlock; -#if ETHR_XCHK - int initialized; -#endif -}; -#endif +#define ETHR_HAVE_ETHR_SIG_FUNCS 1 -/* Static initializers */ -#if ETHR_XCHK -#define ETHR_MUTEX_XCHK_INITER , ETHR_MUTEX_INITIALIZED -#define ETHR_COND_XCHK_INITER , ETHR_COND_INITIALIZED -#else -#define ETHR_MUTEX_XCHK_INITER -#define ETHR_COND_XCHK_INITER +#if defined(PURIFY) || defined(VALGRIND) +# define ETHR_FORCE_PTHREAD_RWLOCK +# define ETHR_FORCE_PTHREAD_MUTEX #endif -#define ETHR_MUTEX_INITER {PTHREAD_MUTEX_INITIALIZER, 0, NULL, NULL ETHR_MUTEX_XCHK_INITER} -#define ETHR_COND_INITER {PTHREAD_COND_INITIALIZER ETHR_COND_XCHK_INITER} - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) \ - || defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) -# define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 -# ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP -# define ETHR_REC_MUTEX_INITER \ - {PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, 1, NULL, NULL ETHR_MUTEX_XCHK_INITER} -# endif -#else -# undef ETHR_HAVE_ETHR_REC_MUTEX_INIT +#if !defined(ETHR_FORCE_PTHREAD_RWLOCK) +# define ETHR_USE_OWN_RWMTX_IMPL__ #endif -#ifndef ETHR_HAVE_PTHREAD_ATFORK -# define ETHR_NO_FORKSAFETY 1 +#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0 +# define ETHR_USE_OWN_MTX_IMPL__ #endif -typedef pthread_key_t ethr_tsd_key; - -#define ETHR_HAVE_ETHR_SIG_FUNCS 1 - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - return pthread_mutex_trylock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - return pthread_mutex_lock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - return pthread_mutex_unlock(&mtx->pt_mtx); -} - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_rdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_wrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -#endif /* ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -#endif /* ETHR_TRY_INLINE_FUNCS */ - #elif defined(ETHR_WIN32_THREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The native win32 threads implementation * @@ -273,409 +171,22 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) # undef WIN32_LEAN_AND_MEAN #endif -/* Types */ -typedef long ethr_tid; /* thread id type */ -typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; -#if ETHR_XCHK - int is_rec_mtx; -#endif -} ethr_mutex; - -typedef struct cnd_wait_event__ cnd_wait_event_; +struct ethr_join_data_; +/* Types */ typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; - cnd_wait_event_ *queue; - cnd_wait_event_ *queue_end; -} ethr_cond; - -#define ETHR_USE_RWMTX_FALLBACK - -/* Static initializers */ - -#define ETHR_MUTEX_INITER {0} -#define ETHR_COND_INITER {0} - -#define ETHR_REC_MUTEX_INITER ETHR_MUTEX_INITER - -#define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 + long id; + struct ethr_join_data_ *jdata; +} ethr_tid; /* thread id type */ typedef DWORD ethr_tsd_key; #undef ETHR_HAVE_ETHR_SIG_FUNCS -#ifdef ETHR_TRY_INLINE_FUNCS -int ethr_fake_static_mutex_init(ethr_mutex *mtx); - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return TryEnterCriticalSection(&mtx->cs) ? 0 : EBUSY; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - EnterCriticalSection(&mtx->cs); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - LeaveCriticalSection(&mtx->cs); - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#ifdef ERTS_MIXED_CYGWIN_VC - -/* atomics */ - -#ifdef _MSC_VER -# if _MSC_VER < 1300 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -# endif -# endif -# endif -# if _MSC_VER >= 1400 -# include <intrin.h> -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -#endif - -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 - -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile LONG x___ = 0; \ - (void) _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ -} while (0) - -typedef struct { - volatile LONG value; -} ethr_atomic_t; - -typedef struct { - volatile LONG locked; -} ethr_spinlock_t; - -typedef struct { - volatile LONG counter; -} ethr_rwlock_t; -#define ETHR_WLOCK_FLAG__ (((LONG) 1) << 30) - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - *i = var->value; -#else - *i = InterlockedExchangeAdd(&var->value, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - (void) InterlockedExchangeAdd(&var->value, (LONG) incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = InterlockedExchangeAdd(&var->value, (LONG) i); - *testp += i; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - (void) InterlockedIncrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - (void) InterlockedDecrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedIncrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedDecrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedAnd() provides a full - * memory barrier. - */ - *old = (long) _InterlockedAnd(&var->value, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedOr() provides a full - * memory barrier. - */ - *old = (long) _InterlockedOr(&var->value, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedCompareExchange() provides a full - * memory barrier. - */ - *old = _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) expected); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = (long) InterlockedExchange(&var->value, (LONG) new); - return 0; -} - -/* - * According to msdn InterlockedExchange() provides a full - * memory barrier. - */ - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->locked = (LONG) 0; -#else - (void) InterlockedExchange(&lock->locked, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) -{ - return 0; -} - - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - InterlockedExchange(&lock->locked, (LONG) 0); -#ifdef DEBUG - ETHR_ASSERT(old == 1); -#endif - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) -{ - LONG old; - do { - old = InterlockedExchange(&lock->locked, (LONG) 1); - } while (old != (LONG) 0); - ETHR_COMPILER_BARRIER; - return 0; -} +#define ETHR_USE_OWN_RWMTX_IMPL__ +#define ETHR_USE_OWN_MTX_IMPL__ -/* - * According to msdn InterlockedIncrement, InterlockedDecrement, - * and InterlockedExchangeAdd(), _InterlockedAnd, and _InterlockedOr - * provides full memory barriers. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->counter = (LONG) 0; -#else - (void) InterlockedExchange(&lock->counter, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - InterlockedDecrement(&lock->counter); - ETHR_ASSERT(old != 0); - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - while (1) { - LONG old = InterlockedIncrement(&lock->counter); - if ((old & ETHR_WLOCK_FLAG__) == 0) - break; /* Got read lock */ - /* Restore and wait for writers to unlock */ - old = InterlockedDecrement(&lock->counter); - while (old & ETHR_WLOCK_FLAG__) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); -#endif - } - } - ETHR_COMPILER_BARRIER; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - _InterlockedAnd(&lock->counter, ~ETHR_WLOCK_FLAG__); - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - LONG old; - do { - old = _InterlockedOr(&lock->counter, ETHR_WLOCK_FLAG__); - } while (old & ETHR_WLOCK_FLAG__); - /* We got the write part of the lock; wait for readers to unlock */ - while ((old & ~ETHR_WLOCK_FLAG__) != 0) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); -#endif - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - ETHR_COMPILER_BARRIER; - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* #ifdef ERTS_MIXED_CYGWIN_VC */ +#define ETHR_YIELD() (Sleep(0), 0) #else /* No supported thread lib found */ @@ -687,6 +198,12 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) #endif +#ifdef SIZEOF_LONG +#if SIZEOF_LONG < ETHR_SIZEOF_PTR +#error size of long currently needs to be at least the same as size of void * +#endif +#endif + /* __builtin_expect() is needed by both native atomics code * and the fallback code */ #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) @@ -698,6 +215,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) # if defined(__GNUC__) # if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) # include "gcc/ethread.h" +# elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) +# include "libatomic_ops/ethread.h" # endif # ifndef ETHR_HAVE_NATIVE_ATOMICS # if ETHR_SIZEOF_PTR == 4 @@ -718,115 +237,141 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) # endif # endif # include "gcc/ethread.h" +# include "libatomic_ops/ethread.h" # endif +# elif defined(ETHR_WIN32_THREADS) +# include "win/ethread.h" # endif -#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) */ - -#ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -# undef ETHR_HAVE_NATIVE_ATOMICS -#endif -#ifdef ETHR_HAVE_OPTIMIZED_LOCKS -# undef ETHR_HAVE_NATIVE_LOCKS -#endif +#endif /* !ETHR_DISABLE_NATIVE_IMPLS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") +# endif +# ifndef ETHR_SPIN_BODY +# if defined(__i386__) || defined(__x86_64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory") +# elif defined(__ia64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory") +# elif defined(__sparc__) +# define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad") +# else +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +# endif +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +# ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0) +# endif #endif -typedef struct { - unsigned open; - ethr_mutex mtx; - ethr_cond cnd; -} ethr_gate; +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#ifndef ETHR_HAVE_NATIVE_ATOMICS /* - * Map ethread native atomics to ethread API atomics. + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when our lock based atomic fallback is used, a noop is sufficient. */ -typedef ethr_native_atomic_t ethr_atomic_t; +#define ETHR_MEMORY_BARRIER do { } while (0) +#define ETHR_WRITE_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0) #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -/* - * Map ethread native spinlocks to ethread API spinlocks. - */ -typedef ethr_native_spinlock_t ethr_spinlock_t; -/* - * Map ethread native rwlocks to ethread API rwlocks. - */ -typedef ethr_native_rwlock_t ethr_rwlock_t; +#ifndef ETHR_WRITE_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL #endif - -#ifdef ETHR_USE_RWMTX_FALLBACK -typedef struct { - ethr_mutex mtx; - ethr_cond rcnd; - ethr_cond wcnd; - unsigned readers; - unsigned waiting_readers; - unsigned waiting_writers; -#if ETHR_XCHK - int initialized; +#ifndef ETHR_READ_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER_IS_FULL #endif -} ethr_rwmutex; +#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER #endif -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -typedef long ethr_atomic_t; -#endif +#define ETHR_FATAL_ERROR__(ERR) \ + ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS +ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err); -#if defined(ETHR_WIN32_THREADS) -typedef struct { - CRITICAL_SECTION cs; -} ethr_spinlock_t; -typedef struct { - CRITICAL_SECTION cs; - unsigned counter; -} ethr_rwlock_t; +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif -int ethr_do_spinlock_init(ethr_spinlock_t *lock); -int ethr_do_rwlock_init(ethr_rwlock_t *lock); +#ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +#endif -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) +#ifndef ETHR_YIELD +# if defined(ETHR_HAVE_SCHED_YIELD) +# ifdef ETHR_HAVE_SCHED_H +# include <sched.h> +# endif +# include <errno.h> +# if defined(ETHR_SCHED_YIELD_RET_INT) +# define ETHR_YIELD() (sched_yield() < 0 ? errno : 0) +# else +# define ETHR_YIELD() (sched_yield(), 0) +# endif +# elif defined(ETHR_HAVE_PTHREAD_YIELD) +# if defined(ETHR_PTHREAD_YIELD_RET_INT) +# define ETHR_YIELD() pthread_yield() +# else +# define ETHR_YIELD() (pthread_yield(), 0) +# endif +# else +# define ETHR_YIELD() (ethr_compiler_barrier(), 0) +# endif +#endif + +#include "ethr_optimized_fallbacks.h" -#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -typedef struct { - pthread_spinlock_t spnlck; -} ethr_spinlock_t; typedef struct { - pthread_spinlock_t spnlck; - unsigned counter; -} ethr_rwlock_t; -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) + void *(*thread_create_prepare_func)(void); + void (*thread_create_parent_func)(void *); + void (*thread_create_child_func)(void *); +} ethr_init_data; -#else /* ethr mutex/rwmutex */ +#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL} typedef struct { - ethr_mutex mtx; -} ethr_spinlock_t; + void *(*alloc)(size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); +} ethr_memory_allocator; + +#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL} typedef struct { - ethr_rwmutex rwmtx; -} ethr_rwlock_t; + ethr_memory_allocator std; + ethr_memory_allocator sl; + ethr_memory_allocator ll; +} ethr_memory_allocators; -#endif /* end mutex/rwmutex */ -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ +#define ETHR_MEM_ALLOCS_DEF_INITER__ \ + {ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__} typedef struct { - void *(*alloc)(size_t); - void *(*realloc)(void *, size_t); - void (*free)(void *); - void *(*thread_create_prepare_func)(void); - void (*thread_create_parent_func)(void *); - void (*thread_create_child_func)(void *); -} ethr_init_data; + ethr_memory_allocators mem; + int reader_groups; + int main_threads; +} ethr_late_init_data; -#define ETHR_INIT_DATA_DEFAULT_INITER {malloc, realloc, free, NULL, NULL, NULL} +#define ETHR_LATE_INIT_DATA_DEFAULT_INITER \ + {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0} typedef struct { int detached; /* boolean (default false) */ @@ -835,18 +380,15 @@ typedef struct { #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1} -#if defined(ETHR_CUSTOM_INLINE_FUNC_NAME_) || !defined(ETHR_TRY_INLINE_FUNCS) -# define ETHR_NEED_MTX_PROTOTYPES__ -# define ETHR_NEED_RWMTX_PROTOTYPES__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) # define ETHR_NEED_SPINLOCK_PROTOTYPES__ +# define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ # define ETHR_NEED_ATOMIC_PROTOTYPES__ #endif -#if !defined(ETHR_NEED_RWMTX_PROTOTYPES__) && defined(ETHR_USE_RWMTX_FALLBACK) -# define ETHR_NEED_RWMTX_PROTOTYPES__ -#endif - int ethr_init(ethr_init_data *); +int ethr_late_init(ethr_late_init_data *); int ethr_install_exit_handler(void (*funcp)(void)); int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *); int ethr_thr_join(ethr_tid, void **); @@ -854,65 +396,6 @@ int ethr_thr_detach(ethr_tid); void ethr_thr_exit(void *); ethr_tid ethr_self(void); int ethr_equal_tids(ethr_tid, ethr_tid); -int ethr_mutex_init(ethr_mutex *); -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT -int ethr_rec_mutex_init(ethr_mutex *); -#endif -int ethr_mutex_destroy(ethr_mutex *); -int ethr_mutex_set_forksafe(ethr_mutex *); -int ethr_mutex_unset_forksafe(ethr_mutex *); -#ifdef ETHR_NEED_MTX_PROTOTYPES__ -int ethr_mutex_trylock(ethr_mutex *); -int ethr_mutex_lock(ethr_mutex *); -int ethr_mutex_unlock(ethr_mutex *); -#endif -int ethr_cond_init(ethr_cond *); -int ethr_cond_destroy(ethr_cond *); -int ethr_cond_signal(ethr_cond *); -int ethr_cond_broadcast(ethr_cond *); -int ethr_cond_wait(ethr_cond *, ethr_mutex *); -int ethr_cond_timedwait(ethr_cond *, ethr_mutex *, ethr_timeval *); - -int ethr_rwmutex_init(ethr_rwmutex *); -int ethr_rwmutex_destroy(ethr_rwmutex *); -#ifdef ETHR_NEED_RWMTX_PROTOTYPES__ -int ethr_rwmutex_tryrlock(ethr_rwmutex *); -int ethr_rwmutex_rlock(ethr_rwmutex *); -int ethr_rwmutex_runlock(ethr_rwmutex *); -int ethr_rwmutex_tryrwlock(ethr_rwmutex *); -int ethr_rwmutex_rwlock(ethr_rwmutex *); -int ethr_rwmutex_rwunlock(ethr_rwmutex *); -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -int ethr_atomic_init(ethr_atomic_t *, long); -int ethr_atomic_set(ethr_atomic_t *, long); -int ethr_atomic_read(ethr_atomic_t *, long *); -int ethr_atomic_inctest(ethr_atomic_t *, long *); -int ethr_atomic_dectest(ethr_atomic_t *, long *); -int ethr_atomic_inc(ethr_atomic_t *); -int ethr_atomic_dec(ethr_atomic_t *); -int ethr_atomic_addtest(ethr_atomic_t *, long, long *); -int ethr_atomic_add(ethr_atomic_t *, long); -int ethr_atomic_and_old(ethr_atomic_t *, long, long *); -int ethr_atomic_or_old(ethr_atomic_t *, long, long *); -int ethr_atomic_xchg(ethr_atomic_t *, long, long *); -int ethr_atomic_cmpxchg(ethr_atomic_t *, long, long, long *); -#endif - -#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ -int ethr_spinlock_init(ethr_spinlock_t *); -int ethr_spinlock_destroy(ethr_spinlock_t *); -int ethr_spin_unlock(ethr_spinlock_t *); -int ethr_spin_lock(ethr_spinlock_t *); - -int ethr_rwlock_init(ethr_rwlock_t *); -int ethr_rwlock_destroy(ethr_rwlock_t *); -int ethr_read_unlock(ethr_rwlock_t *); -int ethr_read_lock(ethr_rwlock_t *); -int ethr_write_unlock(ethr_rwlock_t *); -int ethr_write_lock(ethr_rwlock_t *); -#endif int ethr_time_now(ethr_timeval *); int ethr_tsd_key_create(ethr_tsd_key *); @@ -920,13 +403,6 @@ int ethr_tsd_key_delete(ethr_tsd_key); int ethr_tsd_set(ethr_tsd_key, void *); void *ethr_tsd_get(ethr_tsd_key); -int ethr_gate_init(ethr_gate *); -int ethr_gate_destroy(ethr_gate *); -int ethr_gate_close(ethr_gate *); -int ethr_gate_let_through(ethr_gate *, unsigned); -int ethr_gate_wait(ethr_gate *); -int ethr_gate_swait(ethr_gate *, int); - #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #include <signal.h> int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset); @@ -935,534 +411,579 @@ int ethr_sigwait(const sigset_t *set, int *sig); void ethr_compiler_barrier(void); -#ifdef ETHR_TRY_INLINE_FUNCS - -#ifdef ETHR_HAVE_NATIVE_ATOMICS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_init(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_set(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ - *i = ethr_native_atomic_read(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ethr_native_atomic_add(var, incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = ethr_native_atomic_add_return(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - ethr_native_atomic_inc(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - ethr_native_atomic_dec(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_inc_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_dec_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_and_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_or_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = ethr_native_atomic_xchg(var, new); - return 0; -} - -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_cmpxchg(var, new, expected); - return 0; -} +#if defined(ETHR_HAVE_NATIVE_SPINLOCKS) +typedef ethr_native_spinlock_t ethr_spinlock_t; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) +typedef ethr_opt_spinlock_t ethr_spinlock_t; +#elif defined(__WIN32__) +typedef CRITICAL_SECTION ethr_spinlock_t; +#else +typedef pthread_mutex_t ethr_spinlock_t; +#endif -#endif /* ETHR_HAVE_NATIVE_ATOMICS */ +#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ +int ethr_spinlock_init(ethr_spinlock_t *); +int ethr_spinlock_destroy(ethr_spinlock_t *); +void ethr_spin_unlock(ethr_spinlock_t *); +void ethr_spin_lock(ethr_spinlock_t *); +#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spinlock_init(lock); return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX)) + return ethr_win_get_errno__(); + return 0; +#else + return pthread_mutex_init((pthread_mutex_t *) lock, NULL); +#endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + DeleteCriticalSection((CRITICAL_SECTION *) lock); + return 0; +#else + return pthread_mutex_destroy((pthread_mutex_t *) lock); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_unlock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + LeaveCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_unlock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ - ethr_native_rwlock_init(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_read_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - ethr_native_read_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_write_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - ethr_native_write_lock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + EnterCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_lock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -#endif /* ETHR_HAVE_NATIVE_LOCKS */ - #endif /* ETHR_TRY_INLINE_FUNCS */ +#ifdef ETHR_HAVE_NATIVE_ATOMICS /* - * Fallbacks for atomics used in absence of optimized implementation. + * Map ethread native atomics to ethread API atomics. */ -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS +typedef ethr_native_atomic_t ethr_atomic_t; +#else +typedef long ethr_atomic_t; +#endif +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +void ethr_atomic_init(ethr_atomic_t *, long); +void ethr_atomic_set(ethr_atomic_t *, long); +long ethr_atomic_read(ethr_atomic_t *); +long ethr_atomic_inc_read(ethr_atomic_t *); +long ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +long ethr_atomic_add_read(ethr_atomic_t *, long); +void ethr_atomic_add(ethr_atomic_t *, long); +long ethr_atomic_read_band(ethr_atomic_t *, long); +long ethr_atomic_read_bor(ethr_atomic_t *, long); +long ethr_atomic_xchg(ethr_atomic_t *, long); +long ethr_atomic_cmpxchg(ethr_atomic_t *, long, long); +long ethr_atomic_read_acqb(ethr_atomic_t *); +long ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, long); +void ethr_atomic_dec_relb(ethr_atomic_t *); +long ethr_atomic_dec_read_relb(ethr_atomic_t *); +long ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, long, long); +long ethr_atomic_cmpxchg_relb(ethr_atomic_t *, long, long); +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS /* - * ETHR_MEMORY_BARRIER orders between locked and atomic accesses only, - * i.e. when this atomic fallback is used a noop is sufficient. + * Fallbacks for atomics used in absence of a native implementation. */ -#define ETHR_MEMORY_BARRIER #define ETHR_ATOMIC_ADDR_BITS 10 #define ETHR_ATOMIC_ADDR_SHIFT 6 typedef struct { union { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - pthread_spinlock_t spnlck; -#else - ethr_mutex mtx; -#endif + ethr_spinlock_t lck; char buf[ETHR_CACHE_LINE_SIZE]; } u; } ethr_atomic_protection_t; extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - #define ETHR_ATOMIC_PTR2LCK__(PTR) \ (ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.spnlck) + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) #define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ do { \ - pthread_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = pthread_spin_lock(slp__); \ - if (res__ != 0) \ - return res__; \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ { EXPS; } \ - return pthread_spin_unlock(slp__); \ + ethr_spin_unlock(slp__); \ } while (0) -#else /* ethread mutex */ - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.mtx) - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_mutex *mtxp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(mtxp__); \ - if (res__ != 0) \ - return res__; \ - { EXPS; } \ - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(mtxp__); \ -} while (0) - -#endif /* end ethread mutex */ - -#ifdef ETHR_TRY_INLINE_FUNCS +#endif -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_init(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *i = (long) *var); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) *var); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *incp, long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *testp = (long) ++(*incp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_add(var, incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, long i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_add_return(var, i); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *decp, long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, *testp = (long) --(*decp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_inc(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *incp, - long i, - long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *incp += i; *testp = *incp); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *incp) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, ++(*incp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) ++(*var)); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *decp) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, --(*decp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) --(*var)); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + long mask) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var &= mask); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_and_retold(var, mask); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + long mask) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var |= mask); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_or_retold(var, mask); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif } -static ETHR_INLINE int +static ETHR_INLINE long ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) + long new) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var = new); -} +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_xchg(var, new); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int +static ETHR_INLINE long ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, long new, - long expected, - long *old) + long exp) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__( - var, - long old_val = *var; - *old = old_val; - if (__builtin_expect(old_val == expected, 1)) - *var = new; - ); - return 0; +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg(var, new, exp); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - /* - * Fallbacks for spin locks, and rw spin locks used in absence of - * optimized implementation. + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier */ -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS -#ifdef ETHR_TRY_INLINE_FUNCS +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_init(&lock->spnlck, 0); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return_acqb(var); #else - return ethr_mutex_init(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); #endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, long val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set_relb(var, val); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec_relb(var); #else - return ethr_mutex_destroy(&lock->mtx); + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); #endif } +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return_relb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + long new, + long exp) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_acqb(var, new, exp); #else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); #endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + long new, + long exp) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_lock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_relb(var, new, exp); #else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); #endif } -#ifdef ETHR_USE_RWMTX_FALLBACK -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) X +#endif /* ETHR_TRY_INLINE_FUNCS */ + +typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ + +#if defined(ETHR_WIN32_THREADS) +# include "win/ethr_event.h" #else -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) ETHR_INLINE_FUNC_NAME_(X) +# include "pthread/ethr_event.h" +#endif + +int ethr_set_main_thr_status(int, int); +int ethr_get_main_thr_status(int *); + +struct ethr_ts_event_ { + ethr_ts_event *next; + ethr_ts_event *prev; + ethr_event event; + void *udata; + ethr_atomic_t uaflgs; + unsigned uflgs; + unsigned iflgs; /* for ethr lib only */ + short rgix; /* for ethr lib only */ + short mtix; /* for ethr lib only */ +}; + +#define ETHR_TS_EV_ETHREAD (((unsigned) 1) << 0) +#define ETHR_TS_EV_INITED (((unsigned) 1) << 1) +#define ETHR_TS_EV_TMP (((unsigned) 1) << 2) +#define ETHR_TS_EV_MAIN_THR (((unsigned) 1) << 3) + +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp); +int ethr_free_ts_event__(ethr_ts_event *tsep); +int ethr_make_ts_event__(ethr_ts_event **tsepp); + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) +ethr_ts_event *ethr_get_ts_event(void); +void ethr_leave_ts_event(ethr_ts_event *); +#endif + +#if defined(ETHR_PTHREADS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) + +extern pthread_key_t ethr_ts_event_key__; + +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) +{ + ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_make_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) +{ + +} + #endif +#elif defined(ETHR_WIN32_THREADS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) + +extern DWORD ethr_ts_event_key__; + +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) +{ + ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_get_tmp_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) +{ + if (tsep->iflgs & ETHR_TS_EV_TMP) { + int res = ethr_free_ts_event__(tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +#endif + +#endif + +#include "ethr_mutex.h" /* Need atomic declarations and tse */ + +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +typedef ethr_native_rwlock_t ethr_rwlock_t; +#else +typedef ethr_rwmutex ethr_rwlock_t; +#endif + +#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__ +int ethr_rwlock_init(ethr_rwlock_t *); +int ethr_rwlock_destroy(ethr_rwlock_t *); +void ethr_read_unlock(ethr_rwlock_t *); +void ethr_read_lock(ethr_rwlock_t *); +void ethr_write_unlock(ethr_rwlock_t *); +void ethr_write_lock(ethr_rwlock_t *); +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_init(&lock->spnlck, 0); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_rwlock_init(lock); + return 0; #else - return ethr_rwmutex_init(&lock->rwmtx); + return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL); #endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + return 0; #else - return ethr_rwmutex_destroy(&lock->rwmtx); + return ethr_rwmutex_destroy((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter--; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_runlock)(&lock->rwmtx); + ethr_rwmutex_runlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int locked = 0; - do { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - if ((lock->counter & ETHR_RWLOCK_WRITERS) == 0) { - lock->counter++; - locked = 1; - } - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } while (!locked); - return 0; +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rlock)(&lock->rwmtx); + ethr_rwmutex_rlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwunlock)(&lock->rwmtx); + ethr_rwmutex_rwunlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - while (1) { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter |= ETHR_RWLOCK_WRITERS; - if (lock->counter == ETHR_RWLOCK_WRITERS) - return 0; - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwlock)(&lock->rwmtx); + ethr_rwmutex_rwlock((ethr_rwmutex *) lock); #endif } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ - -#if defined(ETHR_HAVE_OPTIMIZED_LOCKS) || defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -# define ETHR_HAVE_OPTIMIZED_SPINLOCK -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* #ifndef ETHREAD_H__ */ diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index c9fd87c2f6..5debb44756 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -29,26 +29,54 @@ /* Define if you have pthreads */ #undef ETHR_PTHREADS +/* Define if you need the <nptl/pthread.h> header file. */ +#undef ETHR_NEED_NPTL_PTHREAD_H + /* Define if you have the <pthread.h> header file. */ #undef ETHR_HAVE_PTHREAD_H /* Define if the pthread.h header file is in pthread/mit directory. */ #undef ETHR_HAVE_MIT_PTHREAD_H -/* Define if you have the pthread_mutexattr_settype function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE +/* Define if you have the pthread_spin_lock function. */ +#undef ETHR_HAVE_PTHREAD_SPIN_LOCK -/* Define if you have the pthread_mutexattr_setkind_np function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP +/* Define if you want to force usage of pthread rwlocks */ +#undef ETHR_FORCE_PTHREAD_RWLOCK -/* Define if you have the pthread_atfork function. */ -#undef ETHR_HAVE_PTHREAD_ATFORK +/* Define if you have the pthread_rwlockattr_setkind_np() function. */ +#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP -/* Define if you have the pthread_spin_lock function. */ -#undef ETHR_HAVE_PTHREAD_SPIN_LOCK +/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock + attribute. */ +#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP -/* Define if you have a pthread_rwlock implementation that can be used */ -#undef ETHR_HAVE_PTHREAD_RWLOCK_INIT +/* Define if you have a linux futex implementation. */ +#undef ETHR_HAVE_LINUX_FUTEX + +/* Define if you have gcc atomic operations */ +#undef ETHR_HAVE_GCC_ATOMIC_OPS + +/* Define if you prefer gcc native ethread implementations */ +#undef ETHR_PREFER_GCC_NATIVE_IMPLS + +/* Define if you have the <sched.h> header file. */ +#undef ETHR_HAVE_SCHED_H + +/* Define if you have the sched_yield() function. */ +#undef ETHR_HAVE_SCHED_YIELD + +/* Define if you have the pthread_yield() function. */ +#undef ETHR_HAVE_PTHREAD_YIELD + +/* Define if pthread_yield() returns an int. */ +#undef ETHR_PTHREAD_YIELD_RET_INT + +/* Define if sched_yield() returns an int. */ +#undef ETHR_SCHED_YIELD_RET_INT + +/* Define if you want compatibilty with x86 processors before pentium4. */ +#undef ETHR_PRE_PENTIUM4_COMPAT /* Define if you have the pthread_rwlockattr_setkind_np() function. */ #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP @@ -63,6 +91,15 @@ /* Define if you prefer gcc native ethread implementations */ #undef ETHR_PREFER_GCC_NATIVE_IMPLS +/* Define if you have libatomic_ops atomic operations */ +#undef ETHR_HAVE_LIBATOMIC_OPS + +/* Define if you prefer libatomic_ops native ethread implementations */ +#undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS + +/* Define to the size of AO_t if libatomic_ops is used */ +#undef ETHR_SIZEOF_AO_T + /* Define if you want to turn on extra sanity checking in the ethread library */ #undef ETHR_XCHK diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 775030c8d5..5fe6e23477 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -55,6 +55,7 @@ do { \ volatile long x___ = 0; \ (void) __sync_val_compare_and_swap(&x___, (long) 0, (long) 1); \ } while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) @@ -157,6 +158,24 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) return act; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + return __sync_add_and_fetch(&var->counter, (long) 0); +} + +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return + +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif #endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 90b4c5f773..f28258059f 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -32,8 +32,11 @@ typedef struct { volatile long counter; } ethr_native_atomic_t; -#ifdef __x86_64__ +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) #define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") +#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") +#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") #else #define ETHR_MEMORY_BARRIER \ do { \ @@ -42,7 +45,9 @@ do { \ } while (0) #endif -#ifdef ETHR_TRY_INLINE_FUNCS +#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #ifdef __x86_64__ #define LONG_SUFFIX "q" @@ -158,6 +163,22 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) return tmp; } +/* + * Atomic ops with at least specified barriers. + */ + +#define ethr_native_atomic_read_acqb ethr_native_atomic_read +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) +#define ethr_native_atomic_set_relb ethr_native_atomic_set +#else +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#endif +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #undef LONG_SUFFIX #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index fad8b108fa..ed43e77279 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_I386_ETHREAD_H */ diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h index c009be8ef1..be47f459ce 100644 --- a/erts/include/internal/i386/rwlock.h +++ b/erts/include/internal/i386/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #define ETHR_RWLOCK_OFFSET (1<<24) diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h index 2b4832e26a..0325324895 100644 --- a/erts/include/internal/i386/spinlock.h +++ b/erts/include/internal/i386/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) @@ -46,7 +46,7 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock) * On i386 this needs to be a locked operation * to avoid Pentium Pro errata 66 and 92. */ -#if defined(__x86_64__) +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) __asm__ __volatile__("" : : : "memory"); *(unsigned char*)&lock->lock = 0; #else diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h new file mode 100644 index 0000000000..a6eb43a0bd --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -0,0 +1,292 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__ +#define ETHR_LIBATOMIC_OPS_ATOMIC_H__ + +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \ + || defined(__x86_64__) +#define AO_USE_PENTIUM4_INSTRS +#endif + +#include "atomic_ops.h" + +/* + * libatomic_ops can be downloaded from: + * http://www.hpl.hp.com/research/linux/atomic_ops/ + * + * These operations need to be defined by libatomic_ops; + * otherwise, we won't compile: + * - AO_nop_full() + * - AO_load() + * - AO_store() + * - AO_compare_and_swap() + * + * The `AO_t' type also have to be at least as large as + * `void *' and `long' types. + */ + +#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR +#error The AO_t type is too small +#endif + +typedef struct { + volatile AO_t counter; +} ethr_native_atomic_t; + +#define ETHR_MEMORY_BARRIER AO_nop_full() +#ifdef AO_HAVE_nop_write +# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write() +#else +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_HAVE_nop_read +# define ETHR_READ_MEMORY_BARRIER AO_nop_read() +#else +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_NO_DD_ORDERING +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory") +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_atomic_set(ethr_native_atomic_t *var, long value) +{ + AO_store(&var->counter, (AO_t) value); +} + +static ETHR_INLINE void +ethr_native_atomic_init(ethr_native_atomic_t *var, long value) +{ + ethr_native_atomic_set(var, value); +} + +static ETHR_INLINE long +ethr_native_atomic_read(ethr_native_atomic_t *var) +{ + return (long) AO_load(&var->counter); +} + +static ETHR_INLINE long +ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +{ +#ifdef AO_HAVE_fetch_and_add + return ((long) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; +#else + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp + (AO_t) incr; + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) new; + } +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +{ + (void) ethr_native_atomic_add_return(var, incr); +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_add1 + return ((long) AO_fetch_and_add1(&var->counter)) + 1; +#else + return ethr_native_atomic_add_return(var, 1); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_inc(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_inc_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_sub1 + return ((long) AO_fetch_and_sub1(&var->counter)) - 1; +#else + return ethr_native_atomic_add_return(var, -1); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_dec(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_dec_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp & ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) exp; + } +} + +static ETHR_INLINE long +ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp | ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) exp; + } +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long exp) +{ + long act; + do { + if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + return act; +} + +static ETHR_INLINE long +ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) + return (long) exp; + } +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_load_acquire + return (long) AO_load_acquire(&var->counter); +#else + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_add1_acquire + return ((long) AO_fetch_and_add1_acquire(&var->counter)) + 1; +#else + long res = ethr_native_atomic_add_return(var, 1); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long value) +{ +#ifdef AO_HAVE_store_release + AO_store_release(&var->counter, (AO_t) value); +#else + ETHR_MEMORY_BARRIER; + ethr_native_atomic_set(var, value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_sub1_release + return ((long) AO_fetch_and_sub1_release(&var->counter)) - 1; +#else + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_dec_return(var); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_dec_return_relb(var); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +{ +#ifdef AO_HAVE_compare_and_swap_acquire + long act; + do { + if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + AO_nop_full(); + return act; +#else + long act = ethr_native_atomic_cmpxchg(var, new, exp); + ETHR_MEMORY_BARRIER; + return act; +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +{ +#ifdef AO_HAVE_compare_and_swap_release + long act; + do { + if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + return act; +#else + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_cmpxchg(var, new, exp); +#endif +} + + +#endif + +#endif + +#endif diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h new file mode 100644 index 0000000000..ee73ba73bc --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHREAD_LIBATOMIC_OPS_H__ +#define ETHREAD_LIBATOMIC_OPS_H__ + +#include "ethr_atomic.h" + +#endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index 105d874995..f21f7c9588 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -34,7 +34,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, int i) @@ -205,6 +205,26 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) return old; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return + +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_PPC_ATOMIC_H */ diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index d2a72c3dc1..12efc1b653 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_PPC32_ETHREAD_H */ diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h index 9bdab12826..19ec26ab68 100644 --- a/erts/include/internal/ppc32/rwlock.h +++ b/erts/include/internal/ppc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h index 034c20c143..c8460a3e8a 100644 --- a/erts/include/internal/ppc32/spinlock.h +++ b/erts/include/internal/ppc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h new file mode 100644 index 0000000000..104ec287e0 --- /dev/null +++ b/erts/include/internal/pthread/ethr_event.h @@ -0,0 +1,151 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Linux futex implementation of ethread events ------------------------- */ +#define ETHR_LINUX_FUTEX_IMPL__ + +#include <sys/syscall.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/time.h> + +/* + * Note: Linux futexes operate on 32-bit integers, but + * ethr_native_atomic_t are 64-bits on 64-bit + * platforms. This has to be taken into account. + * Therefore, in each individual value used each + * byte look the same. + */ + +#if ETHR_SIZEOF_PTR == 8 + +#define ETHR_EVENT_OFF_WAITER__ 0xffffffffffffffffL +#define ETHR_EVENT_OFF__ 0x7777777777777777L +#define ETHR_EVENT_ON__ 0L + +#elif ETHR_SIZEOF_PTR == 4 + +#define ETHR_EVENT_OFF_WAITER__ 0xffffffffL +#define ETHR_EVENT_OFF__ 0x77777777L +#define ETHR_EVENT_ON__ 0L + +#else + +#error ehrm... + +#endif + +#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE) +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE +#else +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE +#endif + +typedef struct { + ethr_atomic_t futex; +} ethr_event; + +#define ETHR_FUTEX__(FTX, OP, VAL) \ + (-1 == syscall(__NR_futex, (void *) (FTX), (OP), (int) (VAL), NULL, NULL, 0)\ + ? errno : 0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + long val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic_xchg(&e->futex, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic_set(&e->futex, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +typedef struct { + ethr_atomic_t state; + pthread_mutex_t mtx; + pthread_cond_t cnd; +} ethr_event; + +#define ETHR_EVENT_OFF_WAITER__ -1L +#define ETHR_EVENT_OFF__ 1L +#define ETHR_EVENT_ON__ 0L + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + long val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic_xchg(&e->state, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_cond_signal(&e->cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_mutex_unlock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 8fde449a52..2a995d4465 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -32,7 +32,7 @@ typedef struct { __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ : : : "memory") -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #if defined(__arch64__) #define CASX "casx" @@ -172,6 +172,43 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) return new; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + return res; +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ethr_native_atomic_set(var, i); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ethr_native_atomic_dec(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + return ethr_native_atomic_dec_return(var); +} + +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHR_SPARC32_ATOMIC_H */ diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index 1d55399640..dca113b4d6 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_SPARC32_ETHREAD_H */ diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h index 12448e0b06..465ec96866 100644 --- a/erts/include/internal/sparc32/rwlock.h +++ b/erts/include/internal/sparc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h index b4fe48b714..493d514210 100644 --- a/erts/include/internal/sparc32/spinlock.h +++ b/erts/include/internal/sparc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile unsigned char lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 5e4c7ac9fe..69569d82d1 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -34,7 +34,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __insn_mf() -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, long i) @@ -45,7 +45,6 @@ ethr_native_atomic_init(ethr_native_atomic_t *var, long i) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, long i) { - __insn_mf(); atomic_exchange_acq(&var->counter, i); } @@ -58,28 +57,24 @@ ethr_native_atomic_read(ethr_native_atomic_t *var) static ETHR_INLINE void ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) { - __insn_mf(); atomic_add(&var->counter, incr); } static ETHR_INLINE void ethr_native_atomic_inc(ethr_native_atomic_t *var) { - __insn_mf(); atomic_increment(&var->counter); } static ETHR_INLINE void ethr_native_atomic_dec(ethr_native_atomic_t *var) { - __insn_mf(); atomic_decrement(&var->counter); } static ETHR_INLINE long ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) { - __insn_mf(); return atomic_exchange_and_add(&var->counter, incr) + incr; } @@ -98,33 +93,81 @@ ethr_native_atomic_dec_return(ethr_native_atomic_t *var) static ETHR_INLINE long ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_and_val(&var->counter, mask); } static ETHR_INLINE long ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) { - __insn_mf(); return atomic_or_val(&var->counter, mask); } static ETHR_INLINE long ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) { - __insn_mf(); return atomic_exchange_acq(&var->counter, val); } static ETHR_INLINE long ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_inc_return(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long val) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic_set(var, val); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic_dec(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_dec_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +{ + return ethr_native_atomic_cmpxchg(var, new, exp); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_cmpxchg(var, new, exp); +} + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_TILE_ATOMIC_H */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h new file mode 100644 index 0000000000..500459dd6c --- /dev/null +++ b/erts/include/internal/win/ethr_atomic.h @@ -0,0 +1,244 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support when using VC++ + * Author: Rickard Green + */ + +#ifndef ETHR_WIN_ATOMIC_H__ +#define ETHR_WIN_ATOMIC_H__ + +#ifdef _MSC_VER +# if _MSC_VER < 1300 +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ +# else +# if defined(_M_IX86) +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +# else /* I.e. IA64 */ +# if _MSC_VER >= 1400 +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +# else +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +# endif +# endif +# endif +# if _MSC_VER >= 1400 +# include <intrin.h> +# undef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#pragma intrinsic(_ReadWriteBarrier) +#pragma intrinsic(_InterlockedAnd) +#pragma intrinsic(_InterlockedOr) +#else +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +#endif + +/* + * No configure test checking for _Interlocked*_{acq,rel} and + * Interlocked*{Acquire,Release} have been written yet... + * + * Note, that these are pure optimizations for the itanium + * processor. + */ + +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ +#pragma intrinsic(_InterlockedCompareExchange_acq) +#endif +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL +#pragma intrinsic(_InterlockedCompareExchange_rel) +#endif + + +typedef struct { + volatile LONG value; +} ethr_native_atomic_t; + +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile LONG x___ = 0; \ + _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ +} while (0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +{ + var->value = (LONG) i; +} + +static ETHR_INLINE void +ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + var->value = (LONG) i; +#else + (void) InterlockedExchange(&var->value, (LONG) i); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_read(ethr_native_atomic_t *var) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + return var->value; +#else + return InterlockedExchangeAdd(&var->value, (LONG) 0); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +{ + (void) InterlockedExchangeAdd(&var->value, (LONG) incr); +} + +static ETHR_INLINE long +ethr_native_atomic_add_return(ethr_native_atomic_t *var, long i) +{ + LONG tmp = InterlockedExchangeAdd(&var->value, (LONG) i); + return tmp + i; +} + +static ETHR_INLINE void +ethr_native_atomic_inc(ethr_native_atomic_t *var) +{ + (void) InterlockedIncrement(&var->value); +} + +static ETHR_INLINE void +ethr_native_atomic_dec(ethr_native_atomic_t *var) +{ + (void) InterlockedDecrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +{ + return (long) InterlockedIncrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +{ + return (long) InterlockedDecrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +{ + return (long) _InterlockedAnd(&var->value, mask); +} + +static ETHR_INLINE long +ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +{ + return (long) _InterlockedOr(&var->value, mask); +} + + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +{ + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +} + + +static ETHR_INLINE long +ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +{ + return (long) InterlockedExchange(&var->value, (LONG) new); +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDEXCHANGEADDACQUIRE + return (long) InterlockedExchangeAddAcquire(&var->value, (LONG) 0); +#else + return (long) InterlockedExchangeAdd(&var->value, (LONG) 0); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDINCREMENTACQUIRE + return (long) InterlockedIncrementAcquire(&var->value); +#else + return (long) InterlockedIncrement(&var->value); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +{ + (void) InterlockedExchange(&var->value, (LONG) i); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE + (void) InterlockedDecrementRelease(&var->value); +#else + (void) InterlockedDecrement(&var->value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE + return (long) InterlockedDecrementRelease(&var->value); +#else + return (long) InterlockedDecrement(&var->value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long old) +{ +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ + return (long) _InterlockedCompareExchange_acq(&var->value, (LONG) new, (LONG) old); +#else + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long old) +{ + +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL + return (long) _InterlockedCompareExchange_rel(&var->value, (LONG) new, (LONG) old); +#else + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +#endif +} + +#endif + +#endif diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h new file mode 100644 index 0000000000..af57c20f91 --- /dev/null +++ b/erts/include/internal/win/ethr_event.h @@ -0,0 +1,62 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_EVENT_OFF_WAITER__ ((LONG) -1) +#define ETHR_EVENT_OFF__ ((LONG) 1) +#define ETHR_EVENT_ON__ ((LONG) 0) + +typedef struct { + volatile LONG state; + HANDLE handle; +} ethr_event; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + /* InterlockedExchange() imply a full memory barrier which is important */ + LONG state = InterlockedExchange(&e->state, ETHR_EVENT_ON__); + if (state == ETHR_EVENT_OFF_WAITER__) { + if (!SetEvent(e->handle)) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + /* InterlockedExchange() imply a full memory barrier which is important */ + InterlockedExchange(&e->state, ETHR_EVENT_OFF__); +} + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h new file mode 100644 index 0000000000..b52710f6a3 --- /dev/null +++ b/erts/include/internal/win/ethread.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomic and spinlock ethread support when using VC++ + * Author: Rickard Green + */ + +#ifndef ETHREAD_WIN_H__ +#define ETHREAD_WIN_H__ + +#include "ethr_atomic.h" +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index e7caac8072..0d3181cace 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -280,8 +280,13 @@ endif # # ethread library # +ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@ ifneq ($(strip $(ETHR_LIB_NAME)),) -ETHREAD_LIB_SRC=common/ethread.c +ETHREAD_LIB_SRC=common/ethr_aux.c \ + common/ethr_mutex.c \ + common/ethr_cbf.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethr_event.c ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX) ifeq ($(USING_VC),yes) @@ -379,13 +384,13 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(r_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(r_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ $(OBJ_DIR)/%.o: common/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ # Win32 specific @@ -393,25 +398,25 @@ $(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c $(MD_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ -$(MD_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MD_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ $(MDd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ -$(MDd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MDd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ $(MT_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ -$(MT_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MT_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ $(MTd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ -$(MTd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MTd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ # @@ -438,6 +443,8 @@ RELEASE_LIBS=$(ERTS_LIBS) INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/README \ $(ERTS_INCL_INT)/ethread.h \ + $(ERTS_INCL_INT)/ethr_mutex.h \ + $(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \ $(ERTS_INCL_INT)/$(TARGET)/ethread.mk \ $(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \ $(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \ @@ -447,7 +454,8 @@ INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/erl_misc_utils.h \ $(ERTS_INCL_INT)/erl_errno.h -INTERNAL_X_RELEASE_INCLUDE_DIRS= i386 x86_64 ppc32 sparc32 sparc64 tile gcc +INTERNAL_X_RELEASE_INCLUDE_DIRS= \ + i386 x86_64 ppc32 sparc32 sparc64 tile gcc pthread win libatomic_ops INTERNAL_RELEASE_LIBS= \ ../lib/internal/README \ diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index f70db86960..d2ef7140a5 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -59,8 +59,25 @@ # endif #endif -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(HAVE_SCHED_xETAFFINITY) # include <sched.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP) \ + (sched_getaffinity((CPUINFOP)->pid, \ + sizeof(cpu_set_t), \ + &(CPUINFOP)->cpuset) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(SETP) \ + (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0) +#elif defined(__WIN32__) +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +# define cpu_set_t DWORD +# define CPU_SETSIZE (sizeof(DWORD)*8) +# define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0) +# define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU))) +# define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU))) +# define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0) +#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity +#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity #endif #ifdef HAVE_PSET_INFO # include <sys/pset.h> @@ -105,25 +122,58 @@ struct erts_cpu_info_t_ { int available; int topology_size; erts_cpu_topology_t *topology; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *affinity_str; char affinity_str_buf[CPU_SETSIZE/4+2]; cpu_set_t cpuset; +#if defined(HAVE_SCHED_xETAFFINITY) pid_t pid; +#endif #elif defined(HAVE_PSET_INFO) processorid_t *cpuids; #endif }; +#if defined(__WIN32__) + +static __forceinline int +get_proc_affinity(erts_cpu_info_t *cpuinfo) +{ + DWORD pamask, samask; + if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) { + cpuinfo->cpuset = (cpu_set_t) pamask; + return 0; + } + else { + cpuinfo->cpuset = (cpu_set_t) 0; + return -erts_get_last_win_errno(); + } +} + +static __forceinline int +set_thr_affinity(cpu_set_t *set) +{ + if (*set == (cpu_set_t) 0) + return -ENOTSUP; + if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0) + return -erts_get_last_win_errno(); + else + return 0; +} + +#endif + erts_cpu_info_t * erts_cpu_info_create(void) { erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t)); if (!cpuinfo) return NULL; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) cpuinfo->affinity_str = NULL; +#if defined(HAVE_SCHED_xETAFFINITY) cpuinfo->pid = getpid(); +#endif #elif defined(HAVE_PSET_INFO) cpuinfo->cpuids = NULL; #endif @@ -162,10 +212,13 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo) #ifdef __WIN32__ { + int i; SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); cpuinfo->configured = (int) sys_info.dwNumberOfProcessors; - + for (i = 0; i < sizeof(DWORD)*8; i++) + if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i)) + cpuinfo->online++; } #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \ || defined(_SC_NPROCESSORS_ONLN)) @@ -205,8 +258,8 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo) if (cpuinfo->online > cpuinfo->configured) cpuinfo->online = cpuinfo->configured; -#ifdef HAVE_SCHED_xETAFFINITY - if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) { +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo) == 0) { int i, c, cn, si; c = cn = 0; si = sizeof(cpuinfo->affinity_str_buf) - 1; @@ -289,7 +342,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo) char * erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) if (!cpuinfo) return "false"; return cpuinfo->affinity_str; @@ -303,7 +356,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no) { if (!cpuinfo || no < 1 || cpuinfo->available < no) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t *allowed = &cpuinfo->cpuset; int ix, n; @@ -335,8 +388,8 @@ int erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id) { if (cpuinfo && 0 <= id) { -#ifdef HAVE_SCHED_xETAFFINITY - if (id <= CPU_SETSIZE) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (id < CPU_SETSIZE) return CPU_ISSET(id, &cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) int no; @@ -388,7 +441,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) */ if (!cpuinfo) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t bind_set; if (cpu < 0) @@ -398,9 +451,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) CPU_ZERO(&bind_set); CPU_SET(cpu, &bind_set); - if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&bind_set); } #elif defined(HAVE_PROCESSOR_BIND) if (cpu < 0) @@ -418,10 +469,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) { if (!cpuinfo) return -EINVAL; -#if defined(HAVE_SCHED_xETAFFINITY) - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0) - return -errno; - return 0; +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -434,7 +483,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) int erts_unbind_from_cpu_str(char *str) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *c = str; int cpus = 0; int shft = 0; @@ -486,9 +535,7 @@ erts_unbind_from_cpu_str(char *str) if (!cpus) return -EINVAL; - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -965,3 +1012,91 @@ read_topology(erts_cpu_info_t *cpuinfo) } #endif + +#if defined(__WIN32__) + +int +erts_get_last_win_errno(void) +{ + switch (GetLastError()) { + case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ + case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ + case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ + case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ + case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ + case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ + case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ + case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ + case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ + case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ + case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ + case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ + case ERROR_INVALID_DATA: return EINVAL; /* 13 */ + case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ + case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ + case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ + case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ + case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ + case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ + case ERROR_BAD_UNIT: return EACCES; /* 20 */ + case ERROR_NOT_READY: return EACCES; /* 21 */ + case ERROR_BAD_COMMAND: return EACCES; /* 22 */ + case ERROR_CRC: return EACCES; /* 23 */ + case ERROR_BAD_LENGTH: return EACCES; /* 24 */ + case ERROR_SEEK: return EACCES; /* 25 */ + case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ + case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ + case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ + case ERROR_WRITE_FAULT: return EACCES; /* 29 */ + case ERROR_READ_FAULT: return EACCES; /* 30 */ + case ERROR_GEN_FAILURE: return EACCES; /* 31 */ + case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ + case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ + case ERROR_WRONG_DISK: return EACCES; /* 34 */ + case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ + case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ + case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ + case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ + case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ + case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ + case ERROR_FAIL_I24: return EACCES; /* 83 */ + case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ + case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ + case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ + case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ + case ERROR_DISK_FULL: return ENOSPC; /* 112 */ + case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ + case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ + case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ + case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ + case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ + case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ + case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ + case ERROR_NOT_LOCKED: return EACCES; /* 158 */ + case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ + case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ + case ERROR_LOCK_FAILED: return EACCES; /* 167 */ + case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ + case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ + case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ + case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ + case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ + case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ + case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ + case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ + case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ + case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ + case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ + case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ + case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ + case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ + case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ + case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ + case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ + case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ + case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ + default: return EINVAL; + } +} + +#endif diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c new file mode 100644 index 0000000000..4db4cffd3a --- /dev/null +++ b/erts/lib_src/common/ethr_aux.c @@ -0,0 +1,762 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A Thread library for use in the ERTS and other OTP + * applications. + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <stdio.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_AUX_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" +#include <string.h> +#include <limits.h> + +#ifndef __WIN32__ +#include <unistd.h> +#endif + +#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100 +#define ERTS_TS_EV_ALLOC_POOL_SIZE 25 + +erts_cpu_info_t *ethr_cpu_info__; + +int ethr_not_completely_inited__ = 1; +int ethr_not_inited__ = 1; + +ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__; + +#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#endif + +void *(*ethr_thr_prepare_func__)(void) = NULL; +void (*ethr_thr_parent_func__)(void *) = NULL; +void (*ethr_thr_child_func__)(void *) = NULL; + +typedef struct ethr_xhndl_list_ ethr_xhndl_list; +struct ethr_xhndl_list_ { + ethr_xhndl_list *next; + void (*funcp)(void); +}; + +size_t ethr_pagesize__; +size_t ethr_min_stack_size__; /* kilo words */ +size_t ethr_max_stack_size__; /* kilo words */ + +ethr_rwmutex xhndl_rwmtx; +ethr_xhndl_list *xhndl_list; + +static int main_threads; + +static int init_ts_event_alloc(void); + +int +ethr_init_common__(ethr_init_data *id) +{ + int res; + if (id) { + ethr_thr_prepare_func__ = id->thread_create_prepare_func; + ethr_thr_parent_func__ = id->thread_create_parent_func; + ethr_thr_child_func__ = id->thread_create_child_func; + } + + ethr_cpu_info__ = erts_cpu_info_create(); + if (!ethr_cpu_info__) + return ENOMEM; + +#ifdef _SC_PAGESIZE + ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE); +#elif defined(HAVE_GETPAGESIZE) + ethr_pagesize__ = (size_t) getpagesize(); +#else + ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */ +#endif + + /* User needs at least 4 KB */ + ethr_min_stack_size__ = 4*1024; +#if SIZEOF_VOID_P == 8 + /* Double that on 64-bit archs */ + ethr_min_stack_size__ *= 2; +#endif + /* On some systems as much as about 4 KB is used by the system */ + ethr_min_stack_size__ += 4*1024; + /* There should be room for signal handlers */ +#ifdef SIGSTKSZ + ethr_min_stack_size__ += SIGSTKSZ; +#else + ethr_min_stack_size__ += ethr_pagesize__; +#endif + /* The system may think that we need more stack */ +#if defined(PTHREAD_STACK_MIN) + if (ethr_min_stack_size__ < PTHREAD_STACK_MIN) + ethr_min_stack_size__ = PTHREAD_STACK_MIN; +#elif defined(_SC_THREAD_STACK_MIN) + { + size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); + if (ethr_min_stack_size__ < thr_min_stk_sz) + ethr_min_stack_size__ = thr_min_stk_sz; + } +#endif + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ +#ifdef ETHR_STACK_GUARD_SIZE + ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE; +#endif + ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__); + + ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__); + + ethr_max_stack_size__ = 32*1024*1024; +#if SIZEOF_VOID_P == 8 + ethr_max_stack_size__ *= 2; +#endif + ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__); + +#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS + { + int i; + for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { + res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } + } +#endif + + res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__)); + if (res != 0) + return res; + + xhndl_list = NULL; + + return 0; +} + +int +ethr_late_init_common__(ethr_late_init_data *lid) +{ + ethr_ts_event *tsep = NULL; + int reader_groups; + int res; + int i; + ethr_memory_allocator *m[] = {ðr_mem__.std, + ðr_mem__.sl, + ðr_mem__.ll}; + if (lid) + ethr_mem__ = lid->mem; + if (!ethr_mem__.std.alloc + || !ethr_mem__.std.realloc + || !ethr_mem__.std.free) { + ethr_mem__.std.alloc = malloc; + ethr_mem__.std.realloc = realloc; + ethr_mem__.std.free = free; + } + for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) { + if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) { + m[i]->alloc = ethr_mem__.std.alloc; + m[i]->realloc = ethr_mem__.std.realloc; + m[i]->free = ethr_mem__.std.free; + } + + } + res = init_ts_event_alloc(); + if (res != 0) + return res; + res = ethr_make_ts_event__(&tsep); + if (res == 0) + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (!lid) { + main_threads = 0; + reader_groups = 0; + } + else { + if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads) + return res; + main_threads = lid->main_threads; + reader_groups = lid->reader_groups; + } + res = ethr_mutex_lib_late_init(reader_groups, main_threads); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; /* Need it for + rwmutex_init */ + res = ethr_rwmutex_init(&xhndl_rwmtx); + ethr_not_completely_inited__ = 1; + if (res != 0) + return res; + return 0; +} + +int +ethr_install_exit_handler(void (*funcp)(void)) +{ + ethr_xhndl_list *xhp; + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (!funcp) + return EINVAL; + + xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list)); + if (!xhp) + return ENOMEM; + + ethr_rwmutex_rwlock(&xhndl_rwmtx); + + xhp->funcp = funcp; + xhp->next = xhndl_list; + xhndl_list = xhp; + + ethr_rwmutex_rwunlock(&xhndl_rwmtx); + + return 0; +} + +void +ethr_run_exit_handlers__(void) +{ + ethr_xhndl_list *xhp; + + ethr_rwmutex_rlock(&xhndl_rwmtx); + + xhp = xhndl_list; + + ethr_rwmutex_runlock(&xhndl_rwmtx); + + for (; xhp; xhp = xhp->next) + (*xhp->funcp)(); +} + +/* + * Thread specific event alloc, etc. + * + * Note that we don't know when it is safe to destroy an event, but + * we know when it is safe to reuse it. ts_event_free() therefore + * never destroys an event (but makes freed events available for + * reuse). + * + * We could easily keep track of the usage of events, and by this + * make it possible to destroy events. We would however suffer a + * performance penalty for this and save very little memory. + */ + +typedef union { + ethr_ts_event ts_ev; + char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))]; +} ethr_aligned_ts_event; + +static ethr_spinlock_t ts_ev_alloc_lock; +static ethr_ts_event *free_ts_ev; + +#if SIZEOF_VOID_P == SIZEOF_INT +typedef unsigned int EthrPtrSzUInt; +#elif SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long EthrPtrSzUInt; +#else +#error No pointer sized integer type +#endif + +static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp) +{ + int i; + ethr_aligned_ts_event *atsev; + atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size + + ETHR_CACHE_LINE_SIZE); + if (!atsev) + return NULL; + if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0) + atsev = ((ethr_aligned_ts_event *) + ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + for (i = 1; i < size; i++) { + atsev[i-1].ts_ev.next = &atsev[i].ts_ev; + ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0); + atsev[i-1].ts_ev.iflgs = 0; + } + ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0); + atsev[size-1].ts_ev.iflgs = 0; + atsev[size-1].ts_ev.next = NULL; + if (endpp) + *endpp = &atsev[size-1].ts_ev; + return &atsev[0].ts_ev; +} + +static int init_ts_event_alloc(void) +{ + free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE, + NULL); + if (!free_ts_ev) + return ENOMEM; + return ethr_spinlock_init(&ts_ev_alloc_lock); +} + +static ethr_ts_event *ts_event_alloc(void) +{ + ethr_ts_event *ts_ev; + ethr_spin_lock(&ts_ev_alloc_lock); + if (free_ts_ev) { + ts_ev = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + else { + ethr_ts_event *ts_ev_pool_end; + ethr_spin_unlock(&ts_ev_alloc_lock); + + ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end); + if (!ts_ev) + return NULL; + + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev_pool_end->next = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + return ts_ev; +} + +static void ts_event_free(ethr_ts_event *ts_ev) +{ + ETHR_ASSERT(!ts_ev->udata); + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev->next = free_ts_ev; + free_ts_ev = ts_ev; + ethr_spin_unlock(&ts_ev_alloc_lock); +} + +int ethr_make_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED; + tsep->udata = NULL; + tsep->rgix = 0; + tsep->mtix = 0; + + res = ethr_set_tse__(tsep); + if (res != 0 && tsepp && *tsepp) { + ts_event_free(tsep); + return res; + } + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP; + tsep->udata = NULL; + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_free_ts_event__(ethr_ts_event *tsep) +{ + ts_event_free(tsep); + return 0; +} + +void ethr_ts_event_destructor__(void *vtsep) +{ + if (vtsep) { + ethr_ts_event *tsep = (ethr_ts_event *) vtsep; + ts_event_free(tsep); + ethr_set_tse__(NULL); + } +} + +int ethr_set_main_thr_status(int on, int no) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + return EINVAL; + if (on) { + if (no < 1 || main_threads < no) + return EINVAL; + tsep->mtix = (unsigned short) no; + tsep->iflgs |= ETHR_TS_EV_MAIN_THR; + } + else { + tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR; + tsep->mtix = (unsigned short) 0; + } + return 0; +} + +int ethr_get_main_thr_status(int *on) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + *on = 0; + else { + if (tsep->iflgs & ETHR_TS_EV_MAIN_THR) + *on = 1; + else + *on = 0; + } + return 0; +} + + +/* Atomics */ + +void +ethr_atomic_init(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(var); + ethr_atomic_init__(var, i); +} + +void +ethr_atomic_set(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set__(var, i); +} + +long +ethr_atomic_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read__(var); +} + + +long +ethr_atomic_add_read(ethr_atomic_t *var, long incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_add_read__(var, incr); +} + +long +ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read__(var); +} + +long +ethr_atomic_dec_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read__(var); +} + +void +ethr_atomic_add(ethr_atomic_t *var, long incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_add__(var, incr); +} + +void +ethr_atomic_inc(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_inc__(var); +} + +void +ethr_atomic_dec(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec__(var); +} + +long +ethr_atomic_read_band(ethr_atomic_t *var, long mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_band__(var, mask); +} + +long +ethr_atomic_read_bor(ethr_atomic_t *var, long mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_bor__(var, mask); +} + +long +ethr_atomic_xchg(ethr_atomic_t *var, long new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_xchg__(var, new); +} + +long +ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg__(var, new, expected); +} + +long +ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_acqb__(var); +} + +long +ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read_acqb__(var); +} + +void +ethr_atomic_set_relb(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set_relb__(var, i); +} + +void +ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec_relb__(var); +} + +long +ethr_atomic_dec_read_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read_relb__(var); +} + +long +ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_acqb__(var, new, exp); +} + +long +ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_relb__(var, new, exp); +} + + +/* Spinlocks and rwspinlocks */ + +int +ethr_spinlock_init(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_init__(lock); +} + +int +ethr_spinlock_destroy(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_destroy__(lock); +} + +void +ethr_spin_unlock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_unlock__(lock); +} + +void +ethr_spin_lock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_lock__(lock); +} + +int +ethr_rwlock_init(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_init__(lock); +} + +int +ethr_rwlock_destroy(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_destroy__(lock); +} + +void +ethr_read_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_unlock__(lock); +} + +void +ethr_read_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_lock__(lock); +} + +void +ethr_write_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_unlock__(lock); +} + +void +ethr_write_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_lock__(lock); +} + +ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err) +{ + char *errstr; + if (err == ENOTSUP) + errstr = "Operation not supported"; + else { + errstr = strerror(err); + if (!errstr) + errstr = "Unknown error"; + } + fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n", + file, line, func, errstr, err); + ethr_abort__(); +} + +int ethr_assert_failed(const char *file, int line, const char *func, char *a) +{ + fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a); + ethr_abort__(); + return 0; +} diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c new file mode 100644 index 0000000000..04feceec89 --- /dev/null +++ b/erts/lib_src/common/ethr_cbf.c @@ -0,0 +1,36 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +/* + * We keep this function alone in a separate file so the + * compiler wont optimize it away. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ethread.h" + +void +ethr_compiler_barrier_fallback(void) +{ + +} diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c new file mode 100644 index 0000000000..f918bba81d --- /dev/null +++ b/erts/lib_src/common/ethr_mutex.c @@ -0,0 +1,2758 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_MUTEX_IMPL__ + +#include <limits.h> +#include "ethread.h" +#include "ethr_internal.h" + +#define ETHR_SPIN_WITH_WAITERS 1 + +#define ETHR_MTX_MAX_FLGS_SPIN 1000 + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static int default_rwmtx_main_spincount; +static int default_rwmtx_aux_spincount; +#endif +#ifdef ETHR_USE_OWN_MTX_IMPL__ +static int default_mtx_main_spincount; +static int default_mtx_aux_spincount; +static int default_cnd_main_spincount; +static int default_cnd_aux_spincount; +#endif + +static int no_spin; + +#ifndef ETHR_USE_OWN_RWMTX_IMPL__ +static pthread_rwlockattr_t write_pref_attr_data; +static pthread_rwlockattr_t *write_pref_attr; +#endif + +#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__) +# define ETHR_MTX_QLOCK_INIT ethr_spinlock_init +# define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy +# define ETHR_MTX_Q_LOCK ethr_spin_lock +# define ETHR_MTX_Q_UNLOCK ethr_spin_unlock +#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__) +# define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL) +# define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy +# define ETHR_MTX_Q_LOCK(L) \ +do { \ + int res__ = pthread_mutex_lock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +# define ETHR_MTX_Q_UNLOCK(L) \ +do { \ + int res__ = pthread_mutex_unlock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__) +# define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0) +# define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0) +# define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL)) +# define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL)) +#endif + +int +ethr_mutex_lib_init(int cpu_conf) +{ + int res = 0; + + no_spin = cpu_conf == 1; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT; + default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT; + default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT; + default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT; +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT; + default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT; + +#else + +#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \ + && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) + res = pthread_rwlockattr_init(&write_pref_attr_data); + if (res != 0) + return res; + res = pthread_rwlockattr_setkind_np( + &write_pref_attr_data, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + write_pref_attr = &write_pref_attr_data; +#else + write_pref_attr = NULL; +#endif + +#endif + + return res; +} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS +#if 0 /* + * When inc and dec are real atomic instructions as on x86, the + * ETHR_RLOCK_WITH_INC_DEC implementations performs better with + * lots of read locks compared to the cmpxchg based implementation. + * It, however, performs worse with lots of mixed reads and writes. + * It could be used for rwlocks that are known to be read locked + * much, but the readers array based implementation outperforms it + * by far. Therefore, it has been disabled, and will probably be + * removed some time in the future. + */ +# define ETHR_RLOCK_WITH_INC_DEC +#endif +#endif + +static int reader_groups_array_size = 0; +static int main_threads_array_size = 0; + +#endif + +int +ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads) +{ +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + reader_groups_array_size = (no_reader_groups <= 1 + ? 1 + : no_reader_groups + 1); + main_threads_array_size = (no_main_threads <= 1 + ? 1 + : no_main_threads + 1); +#endif + return 0; +} + +int +ethr_rwmutex_set_reader_group(int ix) +{ +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + ethr_ts_event *tse; + + if (ix < 0 || reader_groups_array_size <= ix) + return EINVAL; + + tse = ethr_get_ts_event(); + + if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) { + ethr_leave_ts_event(tse); + return EINVAL; + } + + tse->rgix = ix; + + ethr_leave_ts_event(tse); +#endif + return 0; +} + +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) +static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int); +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0) +#else +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + long initial); +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + long initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock); +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */ + +static ETHR_INLINE void +rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? tse->rgix + : tse->mtix); + rwmtx->tdata.ra[ix].data.waiting_readers++; +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx, + ethr_rwmutex_type type, + int ix, + int inc) +{ + if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0) + ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc); + else { + ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ETHR_ASSERT(inc == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + } +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_inc: + ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_inc; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + } +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec: + ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + return (long) 0; + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + return (long) 0; + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) +{ + long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers); +#ifdef ETHR_DEBUG + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + ETHR_ASSERT(res >= 0); + break; + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + ETHR_ASSERT(res == 0 || res == 1); + break; + default: + ETHR_ASSERT(0); + break; + } +#endif + return res; +} + + +static ETHR_INLINE void +enqueue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (!*queue) { + *queue = tse_start; + tse_start->prev = tse_end; + tse_end->next = tse_start; + } + else { + tse_end->next = *queue; + tse_start->prev = (*queue)->prev; + (*queue)->prev->next = tse_start; + (*queue)->prev = tse_end; + } +} + +static ETHR_INLINE void +insert(ethr_ts_event *tse_pred, ethr_ts_event *tse) +{ + tse->next = tse_pred->next; + tse->prev = tse_pred; + tse_pred->next->prev = tse; + tse_pred->next = tse; +} + +static ETHR_INLINE void +dequeue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (tse_start->prev == tse_end) { + ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start); + *queue = NULL; + } + else { + if (*queue == tse_start) + *queue = tse_end->next; + tse_end->next->prev = tse_start->prev; + tse_start->prev->next = tse_end->next; + } +} + +static void +event_wait(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int spincount, + long type, + int is_rwmtx, + int is_freq_read) +{ + int locked = 0; + long act; + int need_try_complete_runlock = 0; + + /* Need to enqueue and wait... */ + + tse->uflgs = type; + ethr_atomic_set(&tse->uaflgs, type); + + ETHR_MTX_Q_LOCK(&mtxb->qlck); + locked = 1; + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + act = ethr_atomic_read(&mtxb->flgs); + + if (act & type) { + + /* Wait bit already there; enqueue... */ + + ETHR_ASSERT(mtxb->q); + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + enqueue(&mtxb->q, tse, tse); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws++; +#endif + } + else { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(rwmtx->rq_end); + insert(rwmtx->rq_end, tse); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } + } + else { + + /* Set wait bit */ + + while (1) { + long new, exp = act; + int freqread_tryrlock = 0; + need_try_complete_runlock = 0; + + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) + need_try_complete_runlock = 1; + if (act != 0) + new = act | ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; /* Try to get it */ + } + else { + ETHR_ASSERT(is_rwmtx); + + if (!is_freq_read) { + if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__)) + new = act | ETHR_RWMTX_R_WAIT_FLG__; + else + new = act + 1; /* Try to get it */ + } + else { + if (act & ~ETHR_RWMTX_R_FLG__) + new = act | ETHR_RWMTX_R_WAIT_FLG__; + else { /* Try to get it */ + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + rwmutex_freqread_rdrs_inc(rwmtx, tse); + ETHR_MEMORY_BARRIER; + new = act | ETHR_RWMTX_R_FLG__; + freqread_tryrlock = 1; + } + } + } + + act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp); + if (exp == act) { + if (new & type) { + act = new; + break; + } + else { + /* Got it */ + goto done; + } + } + + if (freqread_tryrlock) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + + /* We didn't set ETHR_RWMTX_R_FLG__, however someone + else might have */ + if (act == ETHR_RWMTX_R_FLG__) + goto done; /* Got it by help from someone else */ + + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); + /* + * We know that no waiter flags have been set, i.e., + * we cannot get into a situation where we need to wake + * someone up here. Just restore the readers counter + * and do it over again... + */ + rwmutex_freqread_rdrs_dec(rwmtx, tse); + } + } + + /* Enqueue */ + + if (type == ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(!rwmtx->rq_end); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + else { + mtxb->ws++; + } +#endif + + enqueue(&mtxb->q, tse, tse); + } + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + /* Wait */ + locked = 0; + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + if (need_try_complete_runlock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We were the only one in queue when we enqueued, and it + * was seemingly read locked. We need to try to complete a + * runlock otherwise we might be hanging forever. If the + * runlock could be completed we will be dequeued and + * woken by ourselves. + */ + rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, + act, tse, 0, 1, 0); + } + + while (1) { + ethr_event_reset(&tse->event); + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + + ETHR_ASSERT(act == type); + ethr_event_swait(&tse->event, spincount); + /* swait result: 0 || EINTR */ + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + } + + done: + if (locked) + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); +} + +static void +wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + ethr_ts_event *tse; + + tse = mtxb->q; + ETHR_ASSERT(tse); + dequeue(&mtxb->q, tse, tse); + + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws--; +#endif +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + ethr_atomic_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); +} + +static ETHR_INLINE int +initial_spincount(struct ethr_mutex_base_ *mtxb) +{ + return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN + ? mtxb->aux_scnt + : ETHR_MTX_MAX_FLGS_SPIN); +} + +static ETHR_INLINE int +update_spincount(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int *start_scnt, + int *scnt) +{ + int sscnt = *start_scnt; + if (sscnt < 0) { + *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? mtxb->main_scnt + : mtxb->aux_scnt); + *scnt -= ETHR_MTX_MAX_FLGS_SPIN; + } + else { + *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? mtxb->main_scnt + : mtxb->aux_scnt); + *scnt -= sscnt; + if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) { + *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt; + *start_scnt = -1; + return 0; + } + } + return 1; +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length, + int pre_check); + +static ETHR_INLINE void +write_lock_wait(struct ethr_mutex_base_ *mtxb, + long initial, + int is_rwmtx, + int is_freq_read) +{ + long act = initial; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + int res; + int freq_read_size = -1; + int freq_read_start_ix = -1; + + ETHR_ASSERT(!is_freq_read || is_rwmtx); + + start_scnt = scnt = initial_spincount(mtxb); + + /* + * Spin trying to write lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + long exp; + + while (act != 0) { + + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + if (!tse) + tse = ethr_get_ts_event(); + if (freq_read_size < 0) { + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + freq_read_size = reader_groups_array_size; + freq_read_start_ix = tse->rgix; + } + else { + freq_read_size = main_threads_array_size; + freq_read_start_ix = tse->mtix; + } + } + res = check_readers_array(rwmtx, + freq_read_start_ix, + freq_read_size, + 1); + scnt--; + if (res == 0) { + act = ethr_atomic_read(&mtxb->flgs); + if (act & ETHR_RWMTX_R_MASK__) { + res = rwmutex_try_complete_runlock(rwmtx, act, + tse, 0, 0, + 1); + if (res != EBUSY) + goto done; /* Got it */ + } + if (scnt <= 0) + goto chk_spin; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + continue; + } + } + + if (scnt <= 0) { + chk_spin: + scnt = 0; + + if (!tse) + tse = ethr_get_ts_event(); + if (update_spincount(mtxb, tse, &start_scnt, &scnt)) { + event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__, + is_rwmtx, is_freq_read); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&mtxb->flgs); + scnt--; + } + ETHR_ASSERT(scnt >= 0); + + exp = act; + + act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, + ETHR_RWMTX_W_FLG__, + exp); + if (act == 0) + goto done; /* Got it */ + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static int +mtxb_init(struct ethr_mutex_base_ *mtxb, + int def_main_scnt, + int main_scnt, + int def_aux_scnt, + int aux_scnt) +{ + ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws = 0; +#endif + if (no_spin) { + mtxb->main_scnt = 0; + mtxb->aux_scnt = 0; + } + else { + if (main_scnt > SHRT_MAX) + mtxb->main_scnt = SHRT_MAX; + else if (main_scnt < 0) + mtxb->main_scnt = def_main_scnt; + else + mtxb->main_scnt = (short) main_scnt; + if (aux_scnt > SHRT_MAX) + mtxb->aux_scnt = SHRT_MAX; + else if (aux_scnt < 0) + mtxb->aux_scnt = def_aux_scnt; + else + mtxb->aux_scnt = (short) aux_scnt; + if (mtxb->main_scnt < mtxb->aux_scnt) + mtxb->main_scnt = mtxb->aux_scnt; + + } + mtxb->q = NULL; + ethr_atomic_init(&mtxb->flgs, 0); + return ETHR_MTX_QLOCK_INIT(&mtxb->qlck); +} + +static int +mtxb_destroy(struct ethr_mutex_base_ *mtxb) +{ + long act; + ETHR_MTX_Q_LOCK(&mtxb->qlck); + act = ethr_atomic_read(&mtxb->flgs); + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + if (act != 0) + return EINVAL; + return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck); +} + + +#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Mutex and condition variable implementation * +\* */ + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +/* -- Mutex ---------------------------------------------------------------- */ + +int +ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt) +{ + int res; +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx); + res = mtxb_init(&mtx->mtxb, + default_mtx_main_spincount, + opt ? opt->main_spincount : -1, + default_mtx_aux_spincount, + opt ? opt->aux_spincount : -1); +#if ETHR_XCHK + if (res != 0) + mtx->initialized = 0; +#endif + return res; +} + +int +ethr_mutex_init(ethr_mutex *mtx) +{ + return ethr_mutex_init_opt(mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = 0; +#endif + return mtxb_destroy(&mtx->mtxb); +} + +void +ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial) +{ + write_lock_wait(&mtx->mtxb, initial, 0, 0); +} + +void +ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) +{ + ethr_ts_event *tse; + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + tse = mtx->mtxb.q; + + ETHR_ASSERT(tse); + ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs) + == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__); + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + + /* + * If we have multiple waiters, there is no need to modify + * mtxb->flgs; otherwise, we need to clear the write wait bit... + */ + if (tse->next == mtx->mtxb.q) + ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); + + wake_writer(&mtx->mtxb, 0); +} + +/* -- Condition variables -------------------------------------------------- */ + +static void +enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) +{ + long act; + + /* + * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx' + * is not currently locked by current thread, we almost certainly have a + * hard to debug race condition. There might however be some (strange) + * use for it. POSIX also allow a call to `pthread_cond_signal' or + * `pthread_cond_broadcast' even though the the associated mutex isn't + * locked by the caller. Therefore, we also allow this kind of strange + * usage, but optimize for the case where the mutex is locked by the + * calling thread. + */ + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + { + int dbg_nws__ = 0; + ethr_ts_event *dbg_tse__; + for (dbg_tse__ = tse_start; + dbg_tse__ != tse_end; + dbg_tse__ = dbg_tse__->next) + dbg_nws__++; + mtx->mtxb.ws += dbg_nws__ + 1; + } +#endif + + act = ethr_atomic_read(&mtx->mtxb.flgs); + ETHR_ASSERT(act == 0 + || act == ETHR_RWMTX_W_FLG__ + || act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + if (act & ETHR_RWMTX_W_FLG__) { + /* The normal sane case */ + if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) { + ETHR_ASSERT(!mtx->mtxb.q); + act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, + (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__), + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) { + /* + * Sigh... this wasn't so sane after all since, the mutex was + * obviously not locked by the current thread.... + */ + ETHR_ASSERT(act == 0); + goto mtx_unlocked; + } + } + +#ifdef ETHR_DEBUG + if (act & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_ASSERT(mtx->mtxb.q); + else + ETHR_ASSERT(!mtx->mtxb.q); +#endif + + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + int multi; + mtx_unlocked: + /* Sigh... mutex isn't locked... */ + + multi = tse_start != tse_end; + + while (1) { + long new, exp = act; + + if (multi || (act & ETHR_RWMTX_W_FLG__)) + new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; + + act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp); + if (exp == act) { + ETHR_ASSERT(!mtx->mtxb.q); + if (act & ETHR_RWMTX_W_FLG__) { + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + ETHR_ASSERT(!mtx->mtxb.q); + /* + * Acquired the mutex on behalf of the + * first thread in the queue; wake + * it and enqueue the rest... + */ +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtx->mtxb.ws--; +#endif + if (multi) { + enqueue(&mtx->mtxb.q, tse_start->next, tse_end); + ETHR_ASSERT(mtx->mtxb.q); + } + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + ethr_atomic_set(&tse_start->uaflgs, 0); + ethr_event_set(&tse_start->event); + } + break; + } + } + } +} + +int +ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) +{ + int res; +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd); + cnd->q = NULL; + if (no_spin) { + cnd->main_scnt = 0; + cnd->aux_scnt = 0; + } + else { + if (!opt || opt->main_spincount < 0) + cnd->main_scnt = default_cnd_main_spincount; + else if (opt->main_spincount > SHRT_MAX) + cnd->main_scnt = SHRT_MAX; + else + cnd->main_scnt = (short) opt->main_spincount; + if (!opt || opt->aux_spincount < 0) + cnd->aux_scnt = default_cnd_aux_spincount; + else if (opt->aux_spincount > SHRT_MAX) + cnd->aux_scnt = SHRT_MAX; + else + cnd->aux_scnt = (short) opt->aux_spincount; + if (cnd->main_scnt < cnd->aux_scnt) + cnd->main_scnt = cnd->aux_scnt; + } + ETHR_MTX_QLOCK_INIT(&cnd->qlck); + return 0; +} + +int +ethr_cond_init(ethr_cond *cnd) +{ + return ethr_cond_init_opt(cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__); + + ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + tse->next = tse->prev = NULL; + + enqueue_mtx(mtx, tse, tse); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + } +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int res; + int got_all; + ethr_ts_event *tse; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + do { + got_all = 1; + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + ethr_ts_event *tse_tmp, *tse_end; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = cnd->q->prev; + + tse_tmp = tse; + + do { + + if (mtx == (ethr_mutex *) tse_tmp->udata) { + /* The normal case */ + + ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs) + == ETHR_CND_WAIT_FLG__); + + ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + } + else { + /* Should be very unusual */ + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = tse_tmp->prev; + got_all = 0; + break; + } + + tse_tmp = tse_tmp->next; + + } while (tse_tmp != cnd->q); + + dequeue(&cnd->q, tse, tse_end); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + enqueue_mtx(mtx, tse, tse_end); + } + + } while (!got_all); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int woken; + int scnt; + int res; + void *udata = NULL; + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(!mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + tse = ethr_get_ts_event(); + + scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? cnd->main_scnt + : cnd->aux_scnt); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + udata = tse->udata; /* Got to restore udata before returning */ + tse->udata = (void *) mtx; + + tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */ + ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + enqueue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + ethr_mutex_unlock(mtx); + + /* Wait */ + woken = 0; + while (1) { + long act; + + ethr_event_reset(&tse->event); + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + break; /* Mtx locked */ + + /* First time, got EINTR, or spurious wakeup... */ + + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + + if (woken) { + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued + * on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) { + ETHR_MTX_Q_LOCK(&cnd->qlck); + act = ethr_atomic_read(&tse->uaflgs); + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already + * enqueued on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + if (act == ETHR_CND_WAIT_FLG__) { + tse->udata = udata; + ethr_leave_ts_event(tse); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ethr_mutex_lock(mtx); + return EINTR; + } + } + ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__); + } + ethr_event_swait(&tse->event, scnt); + /* swait result: 0 || EINTR */ + woken = 1; + } + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + tse->udata = udata; + ethr_leave_ts_event(tse); + return 0; +} + +#else +/* -- pthread mutex and condition variables -------------------------------- */ + +int +ethr_mutex_init(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + return pthread_mutex_init(&mtx->pt_mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif +#if ETHR_XCHK + mtx->initialized = 0; +#endif + return pthread_mutex_destroy(&mtx->pt_mtx); +} + +int +ethr_cond_init(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + return pthread_cond_init(&cnd->pt_cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return pthread_cond_destroy(&cnd->pt_cnd); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_signal(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_broadcast(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); + if (res != 0 && res != EINTR) + ETHR_FATAL_ERROR__(res); + return res; +} + +#endif /* pthread_mutex */ + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_mutex_trylock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + return ethr_mutex_trylock__(mtx); +} + +void +ethr_mutex_lock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_lock__(mtx); +} + +void +ethr_mutex_unlock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_unlock__(mtx); +} + + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Read/Write Mutex * +\* */ + +static void +wake_readers(ethr_rwmutex *rwmtx, int rs) +{ + ethr_ts_event *tse; +#ifdef ETHR_DEBUG + int drs = 0; +#endif + + tse = rwmtx->mtxb.q; + ETHR_ASSERT(tse); + ETHR_ASSERT(rwmtx->rq_end); + dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end); + rwmtx->rq_end->next = NULL; + rwmtx->rq_end = NULL; + + ETHR_ASSERT(!rwmtx->mtxb.q + || (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs) + == ETHR_RWMTX_W_WAIT_FLG__)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + + while (tse) { + ethr_ts_event *tse_next; + +#ifdef ETHR_DEBUG + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) + == ETHR_RWMTX_R_WAIT_FLG__); + drs++; +#endif + + tse_next = tse->next; /* we aren't allowed to read tse->next + after we have reset uaflgs */ + + ethr_atomic_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); + tse = tse_next; + } + + ETHR_ASSERT(rs == drs); +} + +static ETHR_INLINE int +is_w_waiter(ethr_ts_event *tse) +{ + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__ + || tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__; +} + +static ETHR_INLINE int +multiple_w_waiters(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(rwmtx->mtxb.q); + ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + + if (!rwmtx->rq_end) + return rwmtx->mtxb.q->next != rwmtx->mtxb.q; + else { + ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q); + if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__) + return 1; + ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q + || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + return rwmtx->rq_end->next != rwmtx->mtxb.q; + } +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length, + int pre_check) +{ + int ix = start_rix; + +#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL + if (pre_check) + ETHR_READ_MEMORY_BARRIER; + else +#endif + ETHR_MEMORY_BARRIER; + + do { + long act = rwmutex_freqread_rdrs_read(rwmtx, ix); + if (act != 0) + return EBUSY; + ix++; + if (ix == length) + ix = 0; + } while (ix != start_rix); + + return 0; +} + +static ETHR_INLINE void +rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + long act; + /* + * Restore failed increment + */ + act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse); + + ETHR_WRITE_MEMORY_BARRIER; + + if (act == 0) { + +#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL + ETHR_READ_MEMORY_BARRIER; +#endif + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if ((act & ETHR_RWMTX_W_FLG__) == 0 + && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) { + /* + * We either got waiters, or someone else trying + * to read unlock which we might have to help. + */ + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0); + } + } +} + +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + long initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock) +{ + ethr_ts_event *tse_tmp; + long act = initial; + int six, res, length; + + tse_tmp = tse; + if (!tse_tmp) + tse_tmp = ethr_get_ts_event(); + + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) + goto check_waiters; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + length = reader_groups_array_size; + six = tse_tmp->rgix; + } + else { + length = main_threads_array_size; + six = tse_tmp->mtix; + } + if (start_next_ix) { + six++; + if (six >= length) + six = 0; + } + + if (!tse) + ethr_leave_ts_event(tse_tmp); + + if (check_before_try) { + res = check_readers_array(rwmtx, six, length, 1); + if (res == EBUSY) + return try_write_lock ? EBUSY : 0; + } + + while (1) { + long exp = act; + long new = act+1; + + ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + < ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + break; + } + if (!try_write_lock) { + if (act == ETHR_RWMTX_W_FLG__ || act == 0) + return 0; + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + return 0; + } + else if ((act & ETHR_RWMTX_R_FLG__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + return 0; + goto check_waiters; + } + } + else { + if (act == 0) + goto tryrwlock; + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) + return EBUSY; + } + } + + res = check_readers_array(rwmtx, six, length, 0); + if (res == EBUSY) { + act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); + if (act & ETHR_RWMTX_R_MASK__) + return try_write_lock ? EBUSY : 0; + } + else { + while (1) { + long exp = act; + long new = act; + new &= ~ETHR_RWMTX_R_FLG__; + new--; + + ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + return try_write_lock ? EBUSY : 0; + act = new; + break; + } + } + } + + /* + * Read unlock completed, but we have to check if + * threads have to be woken (or if we should try + * to write lock it). + */ + + if (act & ETHR_RWMTX_W_FLG__) + return try_write_lock ? EBUSY : 0; + + if (act & ETHR_RWMTX_WAIT_FLGS__) { + check_waiters: + rwmutex_unlock_wake(rwmtx, 0, act); + return try_write_lock ? EBUSY : 0; + } + + if (!try_write_lock) + return 0; + + tryrwlock: + /* Try to write lock it */ + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, + 0); + return act == 0 ? 0 : EBUSY; +} + +#ifdef ETHR_RLOCK_WITH_INC_DEC + +static ETHR_INLINE void +rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx) +{ + long act; + /* + * Restore failed increment + */ + act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + rwmutex_unlock_wake(rwmtx, 0, act); + } +} + +#endif + +static void +rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, + long initial) +{ + long act = initial, exp; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + +#ifdef ETHR_RLOCK_WITH_INC_DEC + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + act = ethr_atomic_read(&rwmtx->mtxb.flgs); +#endif + + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt == 0) { + tse = ethr_get_ts_event(); + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 0); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + scnt--; + } + exp = act; + ETHR_ASSERT(scnt >= 0); + +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs); + if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0) + goto done; /* Got it */ +#else + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) + goto done; /* Got it */ +#endif + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse, + long initial) +{ + long act = initial; + int scnt, start_scnt; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + if (scnt == 0) { + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 1); + return; /* Got it */ + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + scnt--; + } + + ETHR_ASSERT(scnt >= 0); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if (act == ETHR_RWMTX_R_FLG__) + return; /* Got it */ + + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) + break; /* Busy (need to restore inc) */ + + if (act & ETHR_RWMTX_R_FLG__) + return; /* Got it */ + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; /* Got it */ + } + } +} + +static void +rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 0); +} + +static void +rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 1); +} + +static ETHR_INLINE void +rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial) +{ + long act, act_mask; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + /* r pend unlock mask may vary and must be retained */ + act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + } + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + /* rs mask may vary and must be retained */ + act_mask = ETHR_RWMTX_RS_MASK__; +#else + /* rs mask always zero */ + ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0); + ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial); + return; +#endif + } + + act = act_initial; + while (1) { + long exp = act; + long new = new_initial + (act & act_mask); + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +#ifdef ETHR_DEBUG + +static void +dbg_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + ethr_ts_event *tse) +{ + long exp, act, imask; + + exp = have_w ? ETHR_RWMTX_W_FLG__ : 0; + + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + imask = ETHR_RWMTX_RS_MASK__; +#else + imask = 0; +#endif + } + + ETHR_ASSERT(tse); + + if (is_w_waiter(tse)) { + + exp |= ETHR_RWMTX_W_WAIT_FLG__; + if (rwmtx->rq_end) { + exp |= ETHR_RWMTX_R_WAIT_FLG__; + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } + else { + + exp |= ETHR_RWMTX_R_WAIT_FLG__; + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + exp |= ETHR_RWMTX_W_WAIT_FLG__; + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } +} + +#endif + +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) +{ + long new, act = initial; + ethr_ts_event *tse; + + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (!have_w) + return; + else { + while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + long exp = act; + new = exp & ~ETHR_RWMTX_W_FLG__; + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; + } + } + } + + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + tse = rwmtx->mtxb.q; + + if (!have_w) { + if (!tse) { +#ifdef ETHR_DEBUG + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); +#endif + goto already_served; + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act & ~ETHR_RWMTX_WAIT_FLGS__) { + already_served: + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } + } + +#ifdef ETHR_DEBUG + dbg_unlock_wake(rwmtx, have_w, tse); +#endif + + if (is_w_waiter(tse)) { + + if (!have_w) { + act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__); + ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0); + ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__); + act |= ETHR_RWMTX_W_FLG__; + } + + /* + * If we have multiple write waiters, there + * is no need to modify mtxb->flgs; otherwise, + * we need to clear the write wait bit... + */ + if (!multiple_w_waiters(rwmtx)) { + new = ETHR_RWMTX_W_FLG__; + if (tse->next != rwmtx->mtxb.q) { + ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + new |= ETHR_RWMTX_R_WAIT_FLG__; + } + + rwlock_wake_set_flags(rwmtx, new, act); + } + + wake_writer(&rwmtx->mtxb, 1); + } + else { + int rs; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) { + rs = rwmtx->tdata.rs; + new = (long) rs; + rwmtx->tdata.rs = 0; + } + else { + ethr_rwmutex_type type = rwmtx->type; + int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? reader_groups_array_size + : main_threads_array_size); + int ix; + + rs = 0; + for (ix = 0; ix < length; ix++) { + int wrs = rwmtx->tdata.ra[ix].data.waiting_readers; + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + ETHR_ASSERT(wrs >= 0); + if (wrs) { + rs += wrs; + rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs); + } + } + new = ETHR_RWMTX_R_FLG__; + } + + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + new |= ETHR_RWMTX_W_WAIT_FLG__; + + rwlock_wake_set_flags(rwmtx, new, act); + wake_readers(rwmtx, rs); + } +} + +static ethr_rwmtx_readers_array__ * +alloc_readers_array(int length, ethr_rwmutex_lived lived) +{ + ethr_rwmtx_readers_array__ *ra; + size_t sz; + void *mem; + + sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1); + + switch (lived) { + case ETHR_RWMUTEX_LONG_LIVED: + mem = ethr_mem__.ll.alloc(sz); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + mem = ethr_mem__.sl.alloc(sz); + break; + default: + mem = ethr_mem__.std.alloc(sz); + break; + } + if (!mem) + return NULL; + + if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) { + ra = (ethr_rwmtx_readers_array__ *) mem; + ra->data.byte_offset = 0; + } + else { + ra = ((ethr_rwmtx_readers_array__ *) + ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + ra->data.byte_offset = (int) ((unsigned long) ra + - (unsigned long) mem); + } + ra->data.lived = lived; + return ra; +} + +static void +free_readers_array(ethr_rwmtx_readers_array__ *ra) +{ + void *ptr = (void *) (((char *) ra) - ra->data.byte_offset); + switch (ra->data.lived) { + case ETHR_RWMUTEX_LONG_LIVED: + ethr_mem__.ll.free(ptr); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + ethr_mem__.sl.free(ptr); + break; + default: + ethr_mem__.std.free(ptr); + break; + } +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + int res; + ethr_rwmtx_readers_array__ *ra = NULL; +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx); + rwmtx->rq_end = NULL; + rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL; + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + if (main_threads_array_size <= reader_groups_array_size) { + /* No point using reader groups... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ; + } + /* Fall through */ + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + int length; + + length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ + ? main_threads_array_size + : reader_groups_array_size); + + if (length == 1) { + /* No point using a frequent reader type... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL; + } + else { + int ix; + ra = alloc_readers_array(length, + (opt + ? opt->lived + : ETHR_RWMUTEX_UNKNOWN_LIVED)); + if (!ra) { + res = ENOMEM; + goto error; + } + + rwmtx->tdata.ra = ra; + + for (ix = 0; ix < length; ix++) { + ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0); + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + } + break; + } + } + case ETHR_RWMUTEX_TYPE_NORMAL: + rwmtx->tdata.rs = 0; + break; + default: + res = EINVAL; + goto error; + } + res = mtxb_init(&rwmtx->mtxb, + default_rwmtx_main_spincount, + opt ? opt->main_spincount : -1, + default_rwmtx_aux_spincount, + opt ? opt->aux_spincount : -1); + if (res == 0) + return 0; + + error: + + if (ra) + free_readers_array(ra); + +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ + return ethr_rwmutex_init_opt(rwmtx, NULL); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + long act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act == ETHR_RWMTX_R_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0); + } + res = mtxb_destroy(&rwmtx->mtxb); + if (res != 0) + return res; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + free_readers_array(rwmtx->tdata.ra); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return 0; +} + +#define ETHR_MAX_TRYRLOCK_TRIES 5 + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + res = EBUSY; + else { + act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + res = EBUSY; + } + } +#else + long exp = 0; + int tries = 0; + + while (1) { + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) { + res = 0; + break; + } + if (tries > ETHR_MAX_TRYRLOCK_TRIES + || (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) { + res = EBUSY; + break; + } + tries++; + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + res = EBUSY; + break; + } + + if (act & ETHR_RWMTX_R_FLG__) + break; + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + return res; +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + rwmutex_normal_rlock_wait(rwmtx, act); +#else + long exp = 0; + + while (1) { + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) { + break; + } + + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_normal_rlock_wait(rwmtx, act); + break; + } + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + rwmutex_freqread_rlock_wait(rwmtx, tse, act); + break; + } + + if (act & ETHR_RWMTX_R_FLG__) + break; + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + rwmutex_unlock_wake(rwmtx, 0, act); + } + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse); + + ETHR_ASSERT(act >= 0); + + ETHR_WRITE_MEMORY_BARRIER; + + if (act == 0) { + +#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL + ETHR_READ_MEMORY_BARRIER; +#endif + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if ((act & ETHR_RWMTX_W_FLG__) == 0 + && (act & (ETHR_RWMTX_WAIT_FLGS__ + | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) { + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + res = EBUSY; + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + res = 0; + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + do { + + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) { + res = EBUSY; + break; + } + + if (act & ETHR_RWMTX_R_MASK__) { + res = rwmutex_try_complete_runlock(rwmtx, act, NULL, + 0, 1, 1); + break; + } + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + + } while (act != 0); + + break; + } + + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + return res; +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + long act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + rwmutex_normal_rwlock_wait(rwmtx, act); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + do { + + if (act != 0) { + rwmutex_freqread_rwlock_wait(rwmtx, act); + break; + } + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + + } while (act != 0); + + break; + } + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + long act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, + 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act); + break; + } + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +#else +/* -- pthread read/write mutex --------------------------------------------- */ + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ +#if ETHR_XCHK + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr); +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + return ethr_rwmutex_init(rwmtx); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrlock__(rwmtx); +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rlock__(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_runlock__(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrwlock__(rwmtx); +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_rwlock__(rwmtx); +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rwunlock__(rwmtx); +} + +#endif /* pthread */ + + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#ifdef ETHR_MTX_HARD_DEBUG_Q +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int res; + long flgs = ethr_atomic_read(&mtxb->flgs); + + ETHR_MTX_HARD_ASSERT(res == 0); + + ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx); + + if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) { + ETHR_MTX_HARD_ASSERT(!mtxb->q); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(!rwmtx->rs); + } + } + else { + ethr_ts_event *tse; + int ws = 0, rs = 0, rsf = 0, ref = 0; + + ETHR_MTX_HARD_ASSERT(mtxb->q); + + tse = mtxb->q; + + do { + long type; + + ETHR_MTX_HARD_ASSERT(tse->next->prev == tse); + ETHR_MTX_HARD_ASSERT(tse->prev->next == tse); + + type = ethr_atomic_read(&tse->uaflgs); + ETHR_MTX_HARD_ASSERT(type == tse->uflgs); + switch (type) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + if (!rsf) + rsf = 1; + ETHR_MTX_HARD_ASSERT(!ref); + if (rwmtx->rq_end == tse) { + ETHR_MTX_HARD_ASSERT( + tse->next == rwmtx->mtxb.q + || tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ref = 1; + } + rs++; + break; + } + default: + ETHR_MTX_HARD_ASSERT(! "invalid wait type found"); + } + + tse = tse->next; + } while (tse != mtxb->q); + + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs); + } + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + ETHR_MTX_HARD_ASSERT(ws == mtxb->ws); +#endif + + if (flgs & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_MTX_HARD_ASSERT(ws); + else + ETHR_MTX_HARD_ASSERT(!ws); + + if (flgs & ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + ETHR_MTX_HARD_ASSERT(rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(rsf); + ETHR_MTX_HARD_ASSERT(ref); + ETHR_MTX_HARD_ASSERT(rs); + } + else { + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + } + ETHR_MTX_HARD_ASSERT(!rsf); + ETHR_MTX_HARD_ASSERT(!ref); + ETHR_MTX_HARD_ASSERT(!rs); + } + } +} + +#elif defined(ETHR_MTX_HARD_DEBUG_WSQ) + +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int ws = 0; + int rs = 0; + + if (mtxb->q) { + ethr_ts_event *tse = mtxb->q; + do { + switch (tse->uflgs) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: + rs++; + break; + default: + ETHR_MTX_HARD_ASSERT(0); + break; + } + tse = tse->next; + } while (tse != mtxb->q); + } + + ETHR_MTX_HARD_ASSERT(mtxb->ws == ws); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs); + } +} + +#endif + +#endif diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c deleted file mode 100644 index 9c88233934..0000000000 --- a/erts/lib_src/common/ethread.c +++ /dev/null @@ -1,3369 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* - * Description: A Thread library for use in the ERTS and other OTP - * applications. - * Author: Rickard Green - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#undef ETHR_STACK_GUARD_SIZE - -#if defined(ETHR_PTHREADS) - -#ifdef ETHR_TIME_WITH_SYS_TIME -# include <time.h> -# include <sys/time.h> -#else -# ifdef ETHR_HAVE_SYS_TIME_H -# include <sys/time.h> -# else -# include <time.h> -# endif -#endif -#include <sys/types.h> -#include <unistd.h> -#include <signal.h> - -#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE -# define ETHR_STACK_GUARD_SIZE (pagesize) -#endif - -#elif defined(ETHR_WIN32_THREADS) - -#undef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#include <process.h> -#include <winerror.h> - -#else -#error "Missing thread implementation" -#endif - -#include <limits.h> - -#define ETHR_FORCE_INLINE_FUNCS -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ -#include "ethread.h" - -#ifndef ETHR_HAVE_ETHREAD_DEFINES -#error Missing configure defines -#endif - -/* - * ---------------------------------------------------------------------------- - * Common stuff - * ---------------------------------------------------------------------------- - */ - -#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */ - -static int ethr_not_inited = 1; - -#define ASSERT(A) ETHR_ASSERT((A)) - -static void *(*allocp)(size_t) = malloc; -static void *(*reallocp)(void *, size_t) = realloc; -static void (*freep)(void *) = free; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; -#endif - -void *(*thread_create_prepare_func)(void) = NULL; -void (*thread_create_parent_func)(void *) = NULL; -void (*thread_create_child_func)(void *) = NULL; - -typedef struct ethr_xhndl_list_ ethr_xhndl_list; -struct ethr_xhndl_list_ { - ethr_xhndl_list *next; - void (*funcp)(void); -}; - -static size_t pagesize; -#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize) -static size_t min_stack_size; /* kilo words */ -static size_t max_stack_size; /* kilo words */ -#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) -#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) - -ethr_mutex xhndl_mtx; -ethr_xhndl_list *xhndl_list; - -static int -init_common(ethr_init_data *id) -{ - int res; - if (id) { - allocp = id->alloc; - reallocp = id->realloc; - freep = id->free; - thread_create_prepare_func = id->thread_create_prepare_func; - thread_create_parent_func = id->thread_create_parent_func; - thread_create_child_func = id->thread_create_child_func; - } - if (!allocp || !reallocp || !freep) - return EINVAL; - -#ifdef _SC_PAGESIZE - pagesize = (size_t) sysconf(_SC_PAGESIZE); -#elif defined(HAVE_GETPAGESIZE) - pagesize = (size_t) getpagesize(); -#else - pagesize = (size_t) 4*1024; /* Guess 4 KB */ -#endif - - /* User needs at least 4 KB */ - min_stack_size = 4*1024; -#if SIZEOF_VOID_P == 8 - /* Double that on 64-bit archs */ - min_stack_size *= 2; -#endif - /* On some systems as much as about 4 KB is used by the system */ - min_stack_size += 4*1024; - /* There should be room for signal handlers */ -#ifdef SIGSTKSZ - min_stack_size += SIGSTKSZ; -#else - min_stack_size += pagesize; -#endif - /* The system may think that we need more stack */ -#if defined(PTHREAD_STACK_MIN) - if (min_stack_size < PTHREAD_STACK_MIN) - min_stack_size = PTHREAD_STACK_MIN; -#elif defined(_SC_THREAD_STACK_MIN) - { - size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); - if (min_stack_size < thr_min_stk_sz) - min_stack_size = thr_min_stk_sz; - } -#endif - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ -#ifdef ETHR_STACK_GUARD_SIZE - min_stack_size += ETHR_STACK_GUARD_SIZE; -#endif - min_stack_size = ETHR_PAGE_ALIGN(min_stack_size); - - min_stack_size = ETHR_B2KW(min_stack_size); - - max_stack_size = 32*1024*1024; -#if SIZEOF_VOID_P == 8 - max_stack_size *= 2; -#endif - max_stack_size = ETHR_B2KW(max_stack_size); - - xhndl_list = NULL; - - res = ethr_mutex_init(&xhndl_mtx); - if (res != 0) - return res; - - res = ethr_mutex_set_forksafe(&xhndl_mtx); - if (res != 0 && res != ENOTSUP) - return res; - - return 0; -} - -int -ethr_install_exit_handler(void (*funcp)(void)) -{ - ethr_xhndl_list *xhp; - int res; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!funcp) - return EINVAL; - - xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list)); - if (!xhp) - return ENOMEM; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) { - (*freep)((void *) xhp); - return res; - } - - xhp->funcp = funcp; - xhp->next = xhndl_list; - xhndl_list = xhp; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - return res; -} - -static void -run_exit_handlers(void) -{ - int res; - ethr_xhndl_list *xhp; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) - abort(); - - xhp = xhndl_list; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - for (; xhp; xhp = xhp->next) - (*xhp->funcp)(); -} - -#if defined(ETHR_PTHREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * pthread implementation * -\* */ - -typedef struct { - pthread_mutex_t mtx; - pthread_cond_t cnd; - int initialized; - void *(*thr_func)(void *); - void *arg; - void *prep_func_res; -} thr_wrap_data_; - -static int no_ethreads; -static ethr_mutex no_ethrs_mtx; - -#ifndef ETHR_HAVE_PTHREAD_ATFORK -#define ETHR_HAVE_PTHREAD_ATFORK 0 -#endif - -#if !ETHR_HAVE_PTHREAD_ATFORK -#warning "Cannot enforce fork-safety" -#endif - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT -static pthread_rwlockattr_t write_pref_attr_data; -static pthread_rwlockattr_t *write_pref_attr; -#endif - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -/* - * Functions with safe_ prefix aborts on failure. To be used when - * we cannot recover after failure. - */ - -static ETHR_INLINE void -safe_mutex_lock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_lock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_mutex_unlock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_unlock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_cond_signal(pthread_cond_t *cndp) -{ - int res = pthread_cond_signal(cndp); - if (res != 0) - abort(); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static volatile int rec_mtx_attr_need_init = 1; -static pthread_mutexattr_t rec_mtx_attr; - -static int init_rec_mtx_attr(void); - -#endif - -#if ETHR_HAVE_PTHREAD_ATFORK - -static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER; - -static void lock_mutexes(void) -{ - ethr_mutex *m = &forksafe_mtx; - do { - - safe_mutex_lock(&m->pt_mtx); - - m = m->next; - - } while (m != &forksafe_mtx); -} - -static void unlock_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - - safe_mutex_unlock(&m->pt_mtx); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - -static void reinit_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - pthread_mutexattr_t *attrp = NULL; - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - if (m->is_rec_mtx) { - if (rec_mtx_attr_need_init) { - int res = init_rec_mtx_attr(); - if (res != 0) - abort(); - } - attrp = &rec_mtx_attr; - } -#endif - if (pthread_mutex_init(&m->pt_mtx, attrp) != 0) - abort(); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#endif - -static int -init_forksafe(void) -{ - static int init_done = 0; - int res = 0; - - if (init_done) - return res; - - forksafe_mtx.prev = &forksafe_mtx; - forksafe_mtx.next = &forksafe_mtx; - - res = pthread_atfork(lock_mutexes, - unlock_mutexes, -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - reinit_mutexes -#else - unlock_mutexes -#endif - ); - - init_done = 1; - return res; -} - -#endif - - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE); - -#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP); - -#else - -#error "Don't know how to set recursive mutex attributes" - -#endif - -static int -init_rec_mtx_attr(void) -{ - int res, mres; - static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER; - - mres = pthread_mutex_lock(&attrinit_mtx); - if (mres != 0) - return mres; - /* Got here under race conditions; check again ... */ - if (!rec_mtx_attr_need_init) - res = 0; - else { - res = pthread_mutexattr_init(&rec_mtx_attr); - if (res == 0) { - res = SET_REC_MUTEX_ATTR(&rec_mtx_attr); - if (res == 0) - rec_mtx_attr_need_init = 0; - else - (void) pthread_mutexattr_destroy(&rec_mtx_attr); - } - } - - mres = pthread_mutex_unlock(&attrinit_mtx); - if (mres != 0) - return mres; - return res; -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -static ETHR_INLINE void thr_exit_cleanup(void) -{ - run_exit_handlers(); - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); -} - -static void *thr_wrapper(void *vtwd) -{ - void *res; - thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd; - void *(*thr_func)(void *) = twd->thr_func; - void *arg = twd->arg; - - safe_mutex_lock(&twd->mtx); - - if (thread_create_child_func) - (*thread_create_child_func)(twd->prep_func_res); - - twd->initialized = 1; - - safe_cond_signal(&twd->cnd); - safe_mutex_unlock(&twd->mtx); - - res = (*thr_func)(arg); - thr_exit_cleanup(); - return res; -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ - int res; - - if (!ethr_not_inited) - return EINVAL; - - ethr_not_inited = 0; - - res = init_common(id); - if (res != 0) - goto error; - -#if ETHR_HAVE_PTHREAD_ATFORK - init_forksafe(); -#endif - - no_ethreads = 1; - res = ethr_mutex_init(&no_ethrs_mtx); - if (res != 0) - goto error; - res = ethr_mutex_set_forksafe(&no_ethrs_mtx); - if (res != 0 && res != ENOTSUP) - goto error; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { -#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK - res = pthread_spin_init(ðr_atomic_protection__[i].u.spnlck, 0); -#else - res = ethr_mutex_init(ðr_atomic_protection__[i].u.mtx); -#endif - if (res != 0) - goto error; - } - } -#endif - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT -#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \ - && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) - res = pthread_rwlockattr_init(&write_pref_attr_data); - if (res != 0) - goto error; - res = pthread_rwlockattr_setkind_np( - &write_pref_attr_data, - PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - if (res != 0) - goto error; - write_pref_attr = &write_pref_attr_data; -#else - write_pref_attr = NULL; -#endif -#endif - - - return 0; - - error: - ethr_not_inited = 1; - return res; - -} - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - thr_wrap_data_ twd; - pthread_attr_t attr; - int res, dres; - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - - twd.initialized = 0; - twd.thr_func = func; - twd.arg = arg; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Set som thread attributes */ - res = pthread_attr_init(&attr); - if (res != 0) - goto cleanup_parent_func; - res = pthread_mutex_init(&twd.mtx, NULL); - if (res != 0) - goto cleanup_attr_destroy; - res = pthread_cond_init(&twd.cnd, NULL); - if (res != 0) - goto cleanup_mutex_destroy; - - /* Schedule child thread in system scope (if possible) ... */ - res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); - if (res != 0 && res != ENOTSUP) - goto cleanup_cond_destroy; - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; - size_t stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif -#ifdef ETHR_STACK_GUARD_SIZE - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ - suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); -#endif - if (suggested_stack_size < min_stack_size) - stack_size = ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = ETHR_KW2B(max_stack_size); - else - stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - (void) pthread_attr_setstacksize(&attr, stack_size); - } - -#ifdef ETHR_STACK_GUARD_SIZE - (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); -#endif - - /* Detached or joinable... */ - res = pthread_attr_setdetachstate(&attr, - (opts && opts->detached - ? PTHREAD_CREATE_DETACHED - : PTHREAD_CREATE_JOINABLE)); - if (res != 0) - goto cleanup_cond_destroy; - - res = pthread_mutex_lock(&twd.mtx); - - if (res != 0) - goto cleanup_cond_destroy; - - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - if (no_ethreads < ETHR_MAX_THREADS) { - no_ethreads++; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - res = EAGAIN; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - goto cleanup_mutex_unlock; - } - - res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd); - - if (res != 0) { - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - - /* Wait for child to initialize... */ - while (!twd.initialized) { - res = pthread_cond_wait(&twd.cnd, &twd.mtx); - if (res != 0 && res != EINTR) - break; - } - - } - - /* Cleanup... */ - cleanup_mutex_unlock: - dres = pthread_mutex_unlock(&twd.mtx); - if (res == 0) - res = dres; - cleanup_cond_destroy: - dres = pthread_cond_destroy(&twd.cnd); - if (res == 0) - res = dres; - cleanup_mutex_destroy: - dres = pthread_mutex_destroy(&twd.mtx); - if (res == 0) - res = dres; - cleanup_attr_destroy: - dres = pthread_attr_destroy(&attr); - if (res == 0) - res = dres; - cleanup_parent_func: - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - return res; -} - -int -ethr_thr_join(ethr_tid tid, void **res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_join((pthread_t) tid, res); -} - -int -ethr_thr_detach(ethr_tid tid) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_detach((pthread_t) tid); -} - -void -ethr_thr_exit(void *res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - thr_exit_cleanup(); - pthread_exit(res); -} - -ethr_tid -ethr_self(void) -{ - return (ethr_tid) pthread_self(); -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - return pthread_equal((pthread_t) tid1, (pthread_t) tid2); -} - - -/* - * Mutex functions - */ - - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 0; - return pthread_mutex_init(&mtx->pt_mtx, NULL); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - if (rec_mtx_attr_need_init) - init_rec_mtx_attr(); - - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 1; - return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr); -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - if (mtx->next) { - ASSERT(mtx->prev); - ethr_mutex_unset_forksafe(mtx); - } -#if ETHR_XCHK - mtx->initialized = 0; -#endif - return pthread_mutex_destroy(&mtx->pt_mtx); -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (mtx->next) { - /* forksafe already set for this mutex */ - ASSERT(mtx->prev); - } - else { - mtx->next = forksafe_mtx.next; - mtx->prev = &forksafe_mtx; - forksafe_mtx.next->prev = mtx; - forksafe_mtx.next = mtx; - } - - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (!mtx->next) { - /* forksafe already unset for this mutex */ - ASSERT(!mtx->prev); - } - else { - mtx->prev->next = mtx->next; - mtx->next->prev = mtx->prev; - mtx->next = NULL; - mtx->prev = NULL; - } - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = ETHR_COND_INITIALIZED; -#endif - return pthread_cond_init(&cnd->pt_cnd, NULL); -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = 0; -#endif - return pthread_cond_destroy(&cnd->pt_cnd); -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_signal(&cnd->pt_cnd); -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_broadcast(&cnd->pt_cnd); -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - struct timespec to; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !timeout) { - ASSERT(0); - return EINVAL; - } -#endif - - to.tv_sec = timeout->tv_sec; - to.tv_nsec = timeout->tv_nsec; - - return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to); -} - - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr); -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); -#if ETHR_XCHK - rwmtx->initialized = 0; -#endif - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrlock__(rwmtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rlock__(rwmtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_runlock__(rwmtx); -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrwlock__(rwmtx); -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwlock__(rwmtx); -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwunlock__(rwmtx); -} - -#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -/* - * Current time - */ - -int -ethr_time_now(ethr_timeval *time) -{ - int res; - struct timeval tv; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - - res = gettimeofday(&tv, NULL); - time->tv_sec = (long) tv.tv_sec; - time->tv_nsec = ((long) tv.tv_usec)*1000; - return res; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_key_create((pthread_key_t *) keyp, NULL); -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_key_delete((pthread_key_t) key); -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_setspecific((pthread_key_t) key, value); -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return pthread_getspecific((pthread_key_t) key); -} - -/* - * Signal functions - */ - -#if ETHR_HAVE_ETHR_SIG_FUNCS - -int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set && !oset) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_sigmask(how, set, oset); -} - -int ethr_sigwait(const sigset_t *set, int *sig) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set || !sig) { - ASSERT(0); - return EINVAL; - } -#endif - if (sigwait(set, sig) < 0) - return errno; - return 0; -} - -#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ - -#elif defined(ETHR_WIN32_THREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Native win32 threads implementation * -\* */ - -#define INVALID_TID -1 - -/* The spin count values are more or less taken out of the blue */ -#define ETHR_MUTEX_SPIN_COUNT 5000 -#define ETHR_COND_SPIN_COUNT 1000 - -ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */ -ethr_tid last_serial; /* Last thread table serial used */ -ethr_tid last_ix; /* Last thread table index used */ -ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */ - -/* Event used for conditional variables. On per thread. */ -/*typedef struct cnd_wait_event__ cnd_wait_event_;*/ -struct cnd_wait_event__ { - HANDLE handle; - cnd_wait_event_ *prev; - cnd_wait_event_ *next; - int in_queue; -}; - -/* Thread specific data. Stored in the thread table */ -typedef struct { - ethr_tid thr_id; - HANDLE thr_handle; - ethr_tid joiner; - void *result; - cnd_wait_event_ wait_event; -} thr_data_; - -/* Argument passed to thr_wrapper() */ -typedef struct { - void * (*func)(void *); - void * arg; - thr_data_ *ptd; - thr_data_ *td; - int res; - void *prep_func_res; -} thr_wrap_data_; - - -static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect - the thread table from concurrent - accesses. */ -static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect - initialazition of 'statically - initialized' mutexes */ -static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */ - -static DWORD tls_own_thr_data; - -static thr_data_ main_thr_data; - -#define THR_IX(TID) ((TID) & thr_ix_mask) -#define OWN_THR_DATA ((thr_data_ *) TlsGetValue(tls_own_thr_data)) - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -static int -get_errno(void) -{ - switch (GetLastError()) { - case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ - case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ - case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ - case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ - case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ - case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ - case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ - case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ - case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ - case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ - case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ - case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ - case ERROR_INVALID_DATA: return EINVAL; /* 13 */ - case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ - case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ - case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ - case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ - case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ - case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ - case ERROR_BAD_UNIT: return EACCES; /* 20 */ - case ERROR_NOT_READY: return EACCES; /* 21 */ - case ERROR_BAD_COMMAND: return EACCES; /* 22 */ - case ERROR_CRC: return EACCES; /* 23 */ - case ERROR_BAD_LENGTH: return EACCES; /* 24 */ - case ERROR_SEEK: return EACCES; /* 25 */ - case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ - case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ - case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ - case ERROR_WRITE_FAULT: return EACCES; /* 29 */ - case ERROR_READ_FAULT: return EACCES; /* 30 */ - case ERROR_GEN_FAILURE: return EACCES; /* 31 */ - case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ - case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ - case ERROR_WRONG_DISK: return EACCES; /* 34 */ - case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ - case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ - case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ - case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ - case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ - case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ - case ERROR_FAIL_I24: return EACCES; /* 83 */ - case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ - case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ - case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ - case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ - case ERROR_DISK_FULL: return ENOSPC; /* 112 */ - case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ - case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ - case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ - case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ - case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ - case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ - case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ - case ERROR_NOT_LOCKED: return EACCES; /* 158 */ - case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ - case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ - case ERROR_LOCK_FAILED: return EACCES; /* 167 */ - case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ - case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ - case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ - case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ - case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ - case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ - case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ - case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ - case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ - case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ - case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ - case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ - case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ - case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ - case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ - case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ - case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ - case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ - case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ - default: return EINVAL; - } -} - -static ETHR_INLINE thr_data_ * -tid2thr(ethr_tid tid) -{ - ethr_tid ix; - thr_data_ *td; - - if (tid < 0) - return NULL; - ix = THR_IX(tid); - if (ix >= ETHR_MAX_THREADS) - return NULL; - td = thr_table[ix]; - if (!td) - return NULL; - if (td->thr_id != tid) - return NULL; - return td; -} - -static ETHR_INLINE void -new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix) -{ - ethr_tid tmp_serial = last_serial; - ethr_tid tmp_ix = last_ix + 1; - ethr_tid start_ix = tmp_ix; - - - do { - if (tmp_ix >= ETHR_MAX_THREADS) { - tmp_serial++; - if ((tmp_serial << serial_shift) < 0) - tmp_serial = 0; - tmp_ix = 0; - } - if (!thr_table[tmp_ix]) { - *new_tid = (tmp_serial << serial_shift) | tmp_ix; - *new_serial = tmp_serial; - *new_ix = tmp_ix; - return; - } - tmp_ix++; - } while (tmp_ix != start_ix); - - *new_tid = INVALID_TID; - *new_serial = INVALID_TID; - *new_ix = INVALID_TID; - -} - - -static void thr_exit_cleanup(thr_data_ *td, void *res) -{ - - ASSERT(td == OWN_THR_DATA); - - run_exit_handlers(); - - EnterCriticalSection(&thr_table_cs); - CloseHandle(td->wait_event.handle); - if (td->thr_handle == INVALID_HANDLE_VALUE) { - /* We are detached; cleanup thread table */ - ASSERT(td->joiner == INVALID_TID); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - } - else { - /* Save result and let joining thread cleanup */ - td->result = res; - } - LeaveCriticalSection(&thr_table_cs); -} - -static unsigned __stdcall thr_wrapper(LPVOID args) -{ - void *(*func)(void*) = ((thr_wrap_data_ *) args)->func; - void *arg = ((thr_wrap_data_ *) args)->arg; - thr_data_ *td = ((thr_wrap_data_ *) args)->td; - - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE - || !TlsSetValue(tls_own_thr_data, (LPVOID) td)) { - ((thr_wrap_data_ *) args)->res = get_errno(); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - _endthreadex((unsigned) 0); - ASSERT(0); - } - - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - - if (thread_create_child_func) - (*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res); - - ASSERT(td == OWN_THR_DATA); - - ((thr_wrap_data_ *) args)->res = 0; - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - - thr_exit_cleanup(td, (*func)(arg)); - return 0; -} - -int -ethr_fake_static_mutex_init(ethr_mutex *mtx) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!mtx->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, - ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -static int -fake_static_cond_init(ethr_cond *cnd) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!cnd->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, - ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -#ifdef __GNUC__ -#define LL_LITERAL(X) X##LL -#else -#define LL_LITERAL(X) X##i64 -#endif - -#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) - -static ETHR_INLINE void -get_curr_time(long *sec, long *nsec) -{ - SYSTEMTIME t; - FILETIME ft; - LONGLONG lft; - - GetSystemTime(&t); - SystemTimeToFileTime(&t, &ft); - memcpy(&lft, &ft, sizeof(lft)); - *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; - *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); -} - -static cnd_wait_event_ *cwe_freelist; -static CRITICAL_SECTION cwe_cs; - -static int -alloc_cwe(cnd_wait_event_ **cwe_res) -{ - cnd_wait_event_ *cwe; - EnterCriticalSection(&cwe_cs); - cwe = cwe_freelist; - if (cwe) { - cwe_freelist = cwe->next; - LeaveCriticalSection(&cwe_cs); - } - else { - LeaveCriticalSection(&cwe_cs); - cwe = (*allocp)(sizeof(cnd_wait_event_)); - if (!cwe) - return ENOMEM; - cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (cwe->handle == INVALID_HANDLE_VALUE) { - int res = get_errno(); - (*freep)(cwe); - return res; - } - } - *cwe_res = cwe; - return 0; -} - -static -free_cwe(cnd_wait_event_ *cwe) -{ - EnterCriticalSection(&cwe_cs); - cwe->next = cwe_freelist; - cwe_freelist = cwe; - LeaveCriticalSection(&cwe_cs); -} - -static ETHR_INLINE int -condwait(ethr_cond *cnd, - ethr_mutex *mtx, - int with_timeout, - ethr_timeval *timeout) -{ - int res; - thr_data_ *td; - cnd_wait_event_ *cwe; - DWORD code; - long time; /* time until timeout in milli seconds */ - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - - if (!mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || (with_timeout && !timeout)) { - ASSERT(0); - return EINVAL; - } -#endif - - td = OWN_THR_DATA; - if (td) - cwe = &td->wait_event; - else { /* A non-ethread thread */ - res = alloc_cwe(&cwe); - if (res != 0) - return res; - } - - if (!cnd->initialized) - fake_static_cond_init(cnd); - EnterCriticalSection(&cnd->cs); - - ASSERT(!cwe->in_queue); - if (cnd->queue_end) { - ASSERT(cnd->queue); - cwe->prev = cnd->queue_end; - cwe->next = NULL; - cnd->queue_end->next = cwe; - cnd->queue_end = cwe; - } - else { - ASSERT(!cnd->queue); - cwe->prev = NULL; - cwe->next = NULL; - cnd->queue = cwe; - cnd->queue_end = cwe; - } - cwe->in_queue = 1; - - LeaveCriticalSection(&cnd->cs); - - LeaveCriticalSection(&mtx->cs); - - if (!with_timeout) - time = INFINITE; - else { - long sec, nsec; - ASSERT(timeout); - get_curr_time(&sec, &nsec); - time = (timeout->tv_sec - sec)*1000; - time += (timeout->tv_nsec - nsec + 500)/1000000; - if (time < 0) - time = 0; - } - - /* wait for event to signal */ - code = WaitForSingleObject(cwe->handle, time); - - EnterCriticalSection(&mtx->cs); - - if (code == WAIT_OBJECT_0) { - /* We were woken by a signal or a broadcast ... */ - res = 0; - - /* ... no need to remove event from wait queue since this was - taken care of by the signal or broadcast */ -#ifdef DEBUG - EnterCriticalSection(&cnd->cs); - ASSERT(!cwe->in_queue); - LeaveCriticalSection(&cnd->cs); -#endif - - } - else { - /* We timed out... */ - res = ETIMEDOUT; - - /* ... probably have to remove event from wait queue ... */ - EnterCriticalSection(&cnd->cs); - - if (cwe->in_queue) { /* ... but we must check that we are in queue - since a signal or broadcast after timeout - may have removed us from the queue */ - if (cwe->prev) { - cwe->prev->next = cwe->next; - } - else { - ASSERT(cnd->queue == cwe); - cnd->queue = cwe->next; - } - - if (cwe->next) { - cwe->next->prev = cwe->prev; - } - else { - ASSERT(cnd->queue_end == cwe); - cnd->queue_end = cwe->prev; - } - cwe->in_queue = 0; - } - - LeaveCriticalSection(&cnd->cs); - - } - - if (!td) - free_cwe(cwe); - - return res; - -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ -#ifdef _WIN32_WINNT - DWORD major = (_WIN32_WINNT >> 8) & 0xff; - DWORD minor = _WIN32_WINNT & 0xff; - OSVERSIONINFO os_version; -#endif - int err = 0; - thr_data_ *td = &main_thr_data; - unsigned long i; - - if (!ethr_not_inited) - return EINVAL; - -#ifdef _WIN32_WINNT - os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - GetVersionEx(&os_version); - if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT - || os_version.dwMajorVersion < major - || (os_version.dwMajorVersion == major - && os_version.dwMinorVersion < minor)) - return ENOTSUP; -#endif - - ASSERT(ETHR_MAX_THREADS > 0); - for (i = ETHR_MAX_THREADS - 1, serial_shift = 0; - i; - serial_shift++, i >>= 1); - thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift); - - tls_own_thr_data = TlsAlloc(); - if (tls_own_thr_data == TLS_OUT_OF_INDEXES) - goto error; - - last_serial = 0; - last_ix = 0; - - td->thr_id = 0; - td->thr_handle = GetCurrentThread(); - td->joiner = INVALID_TID; - td->result = NULL; - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE) - goto error; - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - thr_table[0] = td; - - if (!TlsSetValue(tls_own_thr_data, (LPVOID) td)) - goto error; - - ASSERT(td == OWN_THR_DATA); - - - cwe_freelist = NULL; - if (!InitializeCriticalSectionAndSpinCount(&cwe_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - - for (i = 1; i < ETHR_MAX_THREADS; i++) - thr_table[i] = NULL; - - if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - ethr_not_inited = 0; - - err = init_common(id); - if (err) - goto error; - - return 0; - - error: - ethr_not_inited = 1; - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - if (td->thr_handle != INVALID_HANDLE_VALUE) - CloseHandle(td->thr_handle); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - return err; -} - -/* - * Thread functions. - */ - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - int err = 0; - thr_wrap_data_ twd; - thr_data_ *my_td, *child_td = NULL; - ethr_tid child_tid, child_serial, child_ix; - DWORD code; - unsigned ID; - unsigned stack_size = 0; /* 0 = system default */ - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - my_td = OWN_THR_DATA; - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif - if (suggested_stack_size < min_stack_size) - stack_size = (unsigned) ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = (unsigned) ETHR_KW2B(max_stack_size); - else - stack_size = - (unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - } - - EnterCriticalSection(&thr_table_cs); - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Find a new thread id to use */ - new_tid(&child_tid, &child_serial, &child_ix); - if (child_tid == INVALID_TID) { - err = EAGAIN; - goto error; - } - - ASSERT(child_ix == THR_IX(child_tid)); - - *tid = child_tid; - - ASSERT(!thr_table[child_ix]); - - /* Alloc thread data */ - thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_)); - if (!child_td) { - err = ENOMEM; - goto error; - } - - /* Init thread data */ - - child_td->thr_id = child_tid; - child_td->thr_handle = INVALID_HANDLE_VALUE; - child_td->joiner = INVALID_TID; - child_td->result = NULL; - /* 'child_td->wait_event' is initialized by child thread */ - - - /* Init thread wrapper data */ - - twd.func = func; - twd.arg = arg; - twd.ptd = my_td; - twd.td = child_td; - twd.res = 0; - - ASSERT(!my_td->wait_event.in_queue); - - /* spawn the thr_wrapper function */ - child_td->thr_handle = (HANDLE) _beginthreadex(NULL, - stack_size, - thr_wrapper, - (LPVOID) &twd, - 0, - &ID); - if (child_td->thr_handle == (HANDLE) 0) { - child_td->thr_handle = INVALID_HANDLE_VALUE; - goto error; - } - - ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE); - - /* Wait for child to finish initialization */ - code = WaitForSingleObject(my_td->wait_event.handle, INFINITE); - if (twd.res || code != WAIT_OBJECT_0) { - err = twd.res; - goto error; - } - - if (opts && opts->detached) { - CloseHandle(child_td->thr_handle); - child_td->thr_handle = INVALID_HANDLE_VALUE; - } - - last_serial = child_serial; - last_ix = child_ix; - - ASSERT(thr_table[child_ix] == child_td); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - if (child_ix != INVALID_TID) { - - if (child_td) { - ASSERT(thr_table[child_ix] == child_td); - - if (child_td->thr_handle != INVALID_HANDLE_VALUE) { - WaitForSingleObject(child_td->thr_handle, INFINITE); - CloseHandle(child_td->thr_handle); - } - - (*freep)((void *) child_td); - thr_table[child_ix] = NULL; - } - } - - *tid = INVALID_TID; - - LeaveCriticalSection(&thr_table_cs); - return err; -} - -int ethr_thr_join(ethr_tid tid, void **res) -{ - int err = 0; - DWORD code; - thr_data_ *td; - thr_data_ *my_td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - my_td = OWN_THR_DATA; - - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - err = ESRCH; - else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone else is joining */ - err = EINVAL; - else if (my_td == td) - err = EDEADLK; - else - td->joiner = my_td->thr_id; - - LeaveCriticalSection(&thr_table_cs); - - if (err) - goto error; - - /* Wait for thread to terminate */ - code = WaitForSingleObject(td->thr_handle, INFINITE); - if (code != WAIT_OBJECT_0) - goto error; - - EnterCriticalSection(&thr_table_cs); - - ASSERT(td == tid2thr(tid)); - ASSERT(td->thr_handle != INVALID_HANDLE_VALUE); - ASSERT(td->joiner == my_td->thr_id); - - if (res) - *res = td->result; - - CloseHandle(td->thr_handle); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - return err; -} - - -int -ethr_thr_detach(ethr_tid tid) -{ - int res; - DWORD code; - thr_data_ *td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!OWN_THR_DATA) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - res = ESRCH; - if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone is joining */ - res = EINVAL; - else { - res = 0; - CloseHandle(td->thr_handle); - td->thr_handle = INVALID_HANDLE_VALUE; - } - - LeaveCriticalSection(&thr_table_cs); - - return res; -} - - -void -ethr_thr_exit(void *res) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - td = OWN_THR_DATA; - if (!td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return; - } - thr_exit_cleanup(td, res); - _endthreadex((unsigned) 0); -} - -ethr_tid -ethr_self(void) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return INVALID_TID; - } -#endif - /* It is okay for non-ethreads (i.e. native win32 threads) to call - ethr_self(). They will however be returned the INVALID_TID. */ - td = OWN_THR_DATA; - if (!td) - return INVALID_TID; - return td->thr_id; -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - /* INVALID_TID does not equal any tid, not even the INVALID_TID */ - return tid1 == tid2 && tid1 != INVALID_TID; -} - -/* - * Mutex functions. - */ - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#if ETHR_XCHK - mtx->is_rec_mtx = 0; -#endif - return 0; -} - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ - int res; - res = ethr_mutex_init(mtx); -#if ETHR_XCHK - mtx->is_rec_mtx = 1; -#endif - return res; -} - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&mtx->cs); - mtx->initialized = 0; - return 0; -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - int res; - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions. - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - return 0; -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || cnd->queue) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&cnd->cs); - cnd->initialized = 0; - return 0; -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - cwe = cnd->queue; - if (cwe) { - ASSERT(cwe->in_queue); - SetEvent(cnd->queue->handle); - if (cwe->next) - cwe->next->prev = NULL; - else { - ASSERT(cnd->queue_end == cnd->queue); - cnd->queue_end = NULL; - } - cnd->queue = cwe->next; - cwe->in_queue = 0; - } - LeaveCriticalSection(&cnd->cs); - return 0; -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - for (cwe = cnd->queue; cwe; cwe = cwe->next) { - ASSERT(cwe->in_queue); - SetEvent(cwe->handle); - cwe->in_queue = 0; - } - cnd->queue = NULL; - cnd->queue_end = NULL; - LeaveCriticalSection(&cnd->cs); - return 0; - -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ - return condwait(cnd, mtx, 0, NULL); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - return condwait(cnd, mtx, 1, timeout); -} - -int -ethr_time_now(ethr_timeval *time) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - get_curr_time(&time->tv_sec, &time->tv_nsec); - return 0; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ - DWORD key; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - key = TlsAlloc(); - if (key == TLS_OUT_OF_INDEXES) - return get_errno(); - *keyp = (ethr_tsd_key) key; - return 0; -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsFree((DWORD) key)) - return get_errno(); - return 0; -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsSetValue((DWORD) key, (LPVOID) value)) - return get_errno(); - return 0; -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return (void *) TlsGetValue((DWORD) key); -} - -/* Misc */ - -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS - -int -ethr_do_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -int -ethr_do_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - lock->counter = 0; - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - -#else -#error "Missing thread implementation" -#endif - -/* Atomics */ - -int -ethr_atomic_init(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_init__(var, i); -} - -int -ethr_atomic_set(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_set__(var, i); -} - -int -ethr_atomic_read(ethr_atomic_t *var, long *i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !i) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_read__(var, i); -} - - -int -ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_addtest__(var, incr, testp); -} - -int -ethr_atomic_inctest(ethr_atomic_t *incp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inctest__(incp, testp); -} - -int -ethr_atomic_dectest(ethr_atomic_t *decp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dectest__(decp, testp); -} - -int -ethr_atomic_add(ethr_atomic_t *var, long incr) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_add__(var, incr); -} - -int -ethr_atomic_inc(ethr_atomic_t *incp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inc__(incp); -} - -int -ethr_atomic_dec(ethr_atomic_t *decp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dec__(decp); -} - -int -ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_and_old__(var, mask, old); -} - -int -ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_or_old__(var, mask, old); -} - -int -ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_xchg__(var, new, old); -} - -int -ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_cmpxchg__(var, new, expected, old); -} - -/* Spinlocks and rwspinlocks */ - -int -ethr_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_init__(lock); -} - -int -ethr_spinlock_destroy(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_destroy__(lock); -} - - -int -ethr_spin_unlock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_unlock__(lock); -} - -int -ethr_spin_lock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_lock__(lock); -} - -int -ethr_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_init__(lock); -} - -int -ethr_rwlock_destroy(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_destroy__(lock); -} - -int -ethr_read_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_unlock__(lock); -} - -int -ethr_read_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_lock__(lock); -} - -int -ethr_write_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_unlock__(lock); -} - -int -ethr_write_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_lock__(lock); -} - - -int -ethr_gate_init(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&gp->mtx); - if (res != 0) - return res; - res = ethr_cond_init(&gp->cnd); - if (res != 0) { - ethr_mutex_destroy(&gp->mtx); - return res; - } - gp->open = 0; - return 0; -} - -int -ethr_gate_destroy(ethr_gate *gp) -{ - int res, dres; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_destroy(&gp->mtx); - dres = ethr_cond_destroy(&gp->cnd); - if (res == 0) - res = dres; - gp->open = 0; - return res; -} - -int -ethr_gate_close(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open = 0; - res = ethr_mutex_unlock__(&gp->mtx); - return res; -} - -int -ethr_gate_let_through(ethr_gate *gp, unsigned no) -{ - int res, ures; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open += no; - res = (gp->open == 1 - ? ethr_cond_signal(&gp->cnd) - : ethr_cond_broadcast(&gp->cnd)); - ures = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - res = ures; - return res; -} - -int -ethr_gate_swait(ethr_gate *gp, int spincount) -{ - int res, ures, n; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - n = spincount; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - while (n >= 0 && !gp->open) { - res = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - return res; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - n--; - } - while (!gp->open) { - res = ethr_cond_wait(&gp->cnd, &gp->mtx); - if (res != 0 && res != EINTR) - goto done; - } - gp->open--; - done: - ures = ethr_mutex_unlock__(&gp->mtx); - if (res == 0) - res = ures; - return res; -} - - -int -ethr_gate_wait(ethr_gate *gp) -{ - return ethr_gate_swait(gp, 0); -} - - -/* rwmutex fallback */ -#ifdef ETHR_USE_RWMTX_FALLBACK - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&rwmtx->mtx); - if (res != 0) - return res; - ethr_cond_init(&rwmtx->rcnd); - if (res != 0) - goto error_cleanup1; - res = ethr_cond_init(&rwmtx->wcnd); - if (res != 0) - goto error_cleanup2; - rwmtx->readers = 0; - rwmtx->waiting_readers = 0; - rwmtx->waiting_writers = 0; -#if ETHR_XCHK - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return 0; - error_cleanup2: - ethr_cond_destroy(&rwmtx->rcnd); - error_cleanup1: - ethr_mutex_destroy(&rwmtx->mtx); - return res; -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res, pres; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = 0; -#endif - res = ethr_mutex_destroy(&rwmtx->mtx); - pres = ethr_cond_destroy(&rwmtx->rcnd); - if (res == 0) - res = pres; - pres = ethr_cond_destroy(&rwmtx->wcnd); - if (res == 0) - res = pres; - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (rwmtx->waiting_writers) { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - while (rwmtx->waiting_writers) { - rwmtx->waiting_readers++; - res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx); - rwmtx->waiting_readers--; - if (res != 0 && res != EINTR) { - (void) ethr_mutex_unlock__(&rwmtx->mtx); - return res; - } - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - rwmtx->readers--; - if (!rwmtx->readers && rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - else { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - - while (rwmtx->readers) { - rwmtx->waiting_writers++; - res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx); - rwmtx->waiting_writers--; - if (res != 0 && res != EINTR) { - (void) ethr_rwmutex_rwunlock(rwmtx); - return res; - } - } - return 0; -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = 0; - if (rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - else if (rwmtx->waiting_readers) - res = ethr_cond_broadcast(&rwmtx->rcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */ - -void -ethr_compiler_barrier(void) -{ - -} - -#ifdef DEBUG - -#include <stdio.h> -int ethr_assert_failed(char *f, int l, char *a) -{ - fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a); - abort(); - return 0; -} - -#endif - - diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c new file mode 100644 index 0000000000..6731c0eb46 --- /dev/null +++ b/erts/lib_src/pthread/ethr_event.c @@ -0,0 +1,219 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#include "ethread.h" + +#if defined(ETHR_LINUX_FUTEX_IMPL__) +/* --- Linux futex implementation of ethread events ------------------------- */ + +#include <sched.h> +#include <errno.h> + +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 + +int +ethr_event_init(ethr_event *e) +{ + ethr_atomic_init(&e->futex, ETHR_EVENT_OFF__); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + unsigned sc = spincount; + int res; + long val; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + while (1) { + val = ethr_atomic_read(&e->futex); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic_cmpxchg(&e->futex, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAIT__, ETHR_EVENT_OFF_WAITER__); + if (res == EINTR) + break; + if (res != 0 && res != EWOULDBLOCK) + ETHR_FATAL_ERROR__(res); + } + + return res; +} + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +int +ethr_event_init(ethr_event *e) +{ + int res; + ethr_atomic_init(&e->state, ETHR_EVENT_OFF__); + res = pthread_mutex_init(&e->mtx, NULL); + if (res != 0) + return res; + res = pthread_cond_init(&e->cnd, NULL); + if (res != 0) { + pthread_mutex_destroy(&e->mtx); + return res; + } + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + int res; + res = pthread_mutex_destroy(&e->mtx); + if (res != 0) + return res; + res = pthread_cond_destroy(&e->cnd); + if (res != 0) + return res; + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + int sc = spincount; + long val; + int res, ulres; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + val = ethr_atomic_read(&e->state); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__ + || val == ETHR_EVENT_OFF__); + + res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + + while (1) { + + val = ethr_atomic_read(&e->state); + if (val == ETHR_EVENT_ON__) + break; + + res = pthread_cond_wait(&e->cnd, &e->mtx); + if (res == EINTR) + break; + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + + ulres = pthread_mutex_unlock(&e->mtx); + if (ulres != 0) + ETHR_FATAL_ERROR__(ulres); + + return res; /* 0 || EINTR */ +} + +#else +#error No ethread event implementation +#endif + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait__(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait__(e, spincount); +} diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c new file mode 100644 index 0000000000..ea1d9d43f0 --- /dev/null +++ b/erts/lib_src/pthread/ethread.c @@ -0,0 +1,477 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Pthread implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#include <stdio.h> +#ifdef ETHR_TIME_WITH_SYS_TIME +# include <time.h> +# include <sys/time.h> +#else +# ifdef ETHR_HAVE_SYS_TIME_H +# include <sys/time.h> +# else +# include <time.h> +# endif +#endif +#include <sys/types.h> +#include <unistd.h> +#include <signal.h> + +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +pthread_key_t ethr_ts_event_key__; +static int child_wait_spin_count; + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(void) +{ + ethr_run_exit_handlers__(); +} + + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_atomic_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +static void *thr_wrapper(void *vtwd) +{ + long result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (long) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(); + return res; +} + +/* internal exports */ + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return pthread_setspecific(ethr_ts_event_key__, (void *) tsep); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return pthread_getspecific(ethr_ts_event_key__); +} + +/* + * -------------------------------------------------------------------------- + * Exported functions + * -------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ + int res; + + if (!ethr_not_inited__) + return EINVAL; + + ethr_not_inited__ = 0; + + res = ethr_init_common__(id); + if (res != 0) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + res = pthread_key_create(ðr_ts_event_key__, ethr_ts_event_destructor__); + + return 0; + error: + ethr_not_inited__ = 1; + return res; + +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + ethr_thr_wrap_data__ twd; + pthread_attr_t attr; + int res, dres; + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + ethr_atomic_init(&twd.result, -1); + twd.tse = ethr_get_ts_event(); + twd.thr_func = func; + twd.arg = arg; + + res = pthread_attr_init(&attr); + if (res != 0) + return res; + + /* Error cleanup needed after this point */ + + /* Schedule child thread in system scope (if possible) ... */ + res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); + if (res != 0 && res != ENOTSUP) + goto error; + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; + size_t stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif +#ifdef ETHR_STACK_GUARD_SIZE + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ + suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + (void) pthread_attr_setstacksize(&attr, stack_size); + } + +#ifdef ETHR_STACK_GUARD_SIZE + (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); +#endif + + /* Detached or joinable... */ + res = pthread_attr_setdetachstate(&attr, + (opts && opts->detached + ? PTHREAD_CREATE_DETACHED + : PTHREAD_CREATE_JOINABLE)); + if (res != 0) + goto error; + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd); + + if (res == 0) { + int spin_count = child_wait_spin_count; + + /* Wait for child to initialize... */ + while (1) { + long result; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + res = (int) result; + goto error; + } + + res = ethr_event_swait(&twd.tse->event, spin_count); + if (res != 0 && res != EINTR) + goto error; + spin_count = 0; + } + } + + /* Cleanup... */ + + error: + dres = pthread_attr_destroy(&attr); + if (res == 0) + res = dres; + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + return res; +} + +int +ethr_thr_join(ethr_tid tid, void **res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_join((pthread_t) tid, res); +} + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_detach((pthread_t) tid); +} + +void +ethr_thr_exit(void *res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + thr_exit_cleanup(); + pthread_exit(res); +} + +ethr_tid +ethr_self(void) +{ + return (ethr_tid) pthread_self(); +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + return pthread_equal((pthread_t) tid1, (pthread_t) tid2); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +/* + * Current time + */ + +int +ethr_time_now(ethr_timeval *time) +{ + int res; + struct timeval tv; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!time) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + res = gettimeofday(&tv, NULL); + time->tv_sec = (long) tv.tv_sec; + time->tv_nsec = ((long) tv.tv_usec)*1000; + return res; +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_key_create((pthread_key_t *) keyp, NULL); +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_key_delete((pthread_key_t) key); +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_setspecific((pthread_key_t) key, value); +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return pthread_getspecific((pthread_key_t) key); +} + +/* + * Signal functions + */ + +#if ETHR_HAVE_ETHR_SIG_FUNCS + +int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set && !oset) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_sigmask(how, set, oset); +} + +int ethr_sigwait(const sigset_t *set, int *sig) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set || !sig) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + if (sigwait(set, sig) < 0) + return errno; + return 0; +} + +#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ + abort(); +} diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c new file mode 100644 index 0000000000..ddb4780ff1 --- /dev/null +++ b/erts/lib_src/win/ethr_event.c @@ -0,0 +1,120 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#include "ethread.h" + +/* --- Windows implementation of thread events ------------------------------ */ + +int +ethr_event_init(ethr_event *e) +{ + e->state = ETHR_EVENT_OFF__; + e->handle = CreateEvent(NULL, FALSE, FALSE, NULL); + if (e->handle == INVALID_HANDLE_VALUE) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + BOOL res = CloseHandle(e->handle); + return res == 0 ? ethr_win_get_errno__() : 0; +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +static ETHR_INLINE int +wait(ethr_event *e, int spincount) +{ + LONG state; + DWORD code; + int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + sc = spincount; + + while (1) { + long on; + while (1) { +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + state = e->state; +#else + state = InterlockedExchangeAdd(&e->state, (LONG) 0); +#endif + if (state == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (state != ETHR_EVENT_OFF_WAITER__) { + state = _InterlockedCompareExchange(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (state == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(state == ETHR_EVENT_OFF__); + } + + code = WaitForSingleObject(e->handle, INFINITE); + if (code != WAIT_OBJECT_0) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } + +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait(e, spincount); +} diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c new file mode 100644 index 0000000000..69523edf94 --- /dev/null +++ b/erts/lib_src/win/ethread.c @@ -0,0 +1,625 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Windows native threads implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <winerror.h> +#include <stdio.h> +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_tid *tid; + ethr_atomic_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +#define ETHR_INVALID_TID_ID -1 + +struct ethr_join_data_ { + HANDLE handle; + void *res; +}; + +static ethr_atomic_t thread_id_counter; +static DWORD own_tid_key; +static ethr_tid main_thr_tid; +static int child_wait_spin_count; + +DWORD ethr_ts_event_key__; + +#define ETHR_GET_OWN_TID__ ((ethr_tid *) TlsGetValue(own_tid_key)) + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(ethr_tid *tid, void *res) +{ + + ETHR_ASSERT(tid == ETHR_GET_OWN_TID__); + + if (tid->jdata) + tid->jdata->res = res; + + ethr_run_exit_handlers__(); + ethr_ts_event_destructor__((void *) ethr_get_tse__()); +} + +static unsigned __stdcall thr_wrapper(LPVOID vtwd) +{ + ethr_tid my_tid; + long result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (long) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + my_tid = *twd->tid; + if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) { + result = (long) ethr_win_get_errno__(); + ethr_free_ts_event__(tsep); + } + else { + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(&my_tid, res); + return 0; +} + +#ifdef __GNUC__ +#define LL_LITERAL(X) X##LL +#else +#define LL_LITERAL(X) X##i64 +#endif + +#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) + +static ETHR_INLINE void +get_curr_time(long *sec, long *nsec) +{ + SYSTEMTIME t; + FILETIME ft; + LONGLONG lft; + + GetSystemTime(&t); + SystemTimeToFileTime(&t, &ft); + memcpy(&lft, &ft, sizeof(lft)); + *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; + *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); +} + +/* internal exports */ + +int +ethr_win_get_errno__(void) +{ + return erts_get_last_win_errno(); +} + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return (TlsSetValue(ethr_ts_event_key__, (LPVOID) tsep) + ? 0 + : ethr_win_get_errno__()); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__); +} + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ +#if 1 + DebugBreak(); +#else + abort(); +#endif +} + +/* + * ---------------------------------------------------------------------------- + * Exported functions + * ---------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ +#ifdef _WIN32_WINNT + DWORD major = (_WIN32_WINNT >> 8) & 0xff; + DWORD minor = _WIN32_WINNT & 0xff; + OSVERSIONINFO os_version; +#endif + int err = 0; + unsigned long i; + + if (!ethr_not_inited__) + return EINVAL; + +#ifdef _WIN32_WINNT + os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + GetVersionEx(&os_version); + if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT + || os_version.dwMajorVersion < major + || (os_version.dwMajorVersion == major + && os_version.dwMinorVersion < minor)) + return ENOTSUP; +#endif + err = ethr_init_common__(id); + if (err) + goto error; + + own_tid_key = TlsAlloc(); + if (own_tid_key == TLS_OUT_OF_INDEXES) + goto error; + + ethr_atomic_init(&thread_id_counter, 0); + + main_thr_tid.id = 0; + main_thr_tid.jdata = NULL; + + if (!TlsSetValue(own_tid_key, (LPVOID) &main_thr_tid)) + goto error; + + ETHR_ASSERT(&main_thr_tid == ETHR_GET_OWN_TID__); + + ethr_ts_event_key__ = TlsAlloc(); + if (ethr_ts_event_key__ == TLS_OUT_OF_INDEXES) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + ethr_not_inited__ = 0; + + return 0; + + error: + ethr_not_inited__ = 1; + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + return err; +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + + +/* + * Thread functions. + */ + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + HANDLE handle = INVALID_HANDLE_VALUE; + int err = 0; + ethr_thr_wrap_data__ twd; + DWORD code; + unsigned ID; + unsigned stack_size = 0; /* 0 = system default */ + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + do { + tid->id = ethr_atomic_inc_read(&thread_id_counter); + } while (tid->id == ETHR_INVALID_TID_ID); + + if (opts && opts->detached) + tid->jdata = NULL; + else { + tid->jdata = ethr_mem__.std.alloc(sizeof(struct ethr_join_data_)); + if (!tid->jdata) + return ENOMEM; + tid->jdata->handle = INVALID_HANDLE_VALUE; + tid->jdata->res = NULL; + } + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = (unsigned) + ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + } + + ethr_atomic_init(&twd.result, -1); + + twd.tid = tid; + twd.thr_func = func; + twd.arg = arg; + twd.tse = ethr_get_ts_event(); + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + /* spawn the thr_wrapper function */ + handle = (HANDLE) _beginthreadex(NULL, stack_size, thr_wrapper, + (LPVOID) &twd, 0, &ID); + if (handle == (HANDLE) 0) { + handle = INVALID_HANDLE_VALUE; + goto error; + } + else { + int spin_count = child_wait_spin_count; + + ETHR_ASSERT(handle != INVALID_HANDLE_VALUE); + + if (!tid->jdata) + CloseHandle(handle); + else + tid->jdata->handle = handle; + + /* Wait for child to initialize... */ + while (1) { + long result; + int err; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + err = (int) result; + goto error; + } + + err = ethr_event_swait(&twd.tse->event, spin_count); + if (err && err != EINTR) + goto error; + spin_count = 0; + } + } + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return 0; + + error: + + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (handle != INVALID_HANDLE_VALUE) { + WaitForSingleObject(handle, INFINITE); + CloseHandle(handle); + } + + if (tid->jdata) { + ethr_mem__.std.free(tid->jdata); + tid->jdata = NULL; + } + + tid->id = ETHR_INVALID_TID_ID; + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return err; +} + +int ethr_thr_join(ethr_tid tid, void **res) +{ + DWORD code; + +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + /* Wait for thread to terminate */ + code = WaitForSingleObject(tid.jdata->handle, INFINITE); + if (code != WAIT_OBJECT_0) + return ethr_win_get_errno__(); + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + if (res) + *res = tid.jdata->res; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +void +ethr_thr_exit(void *res) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ETHR_ASSERT(0); + _endthreadex((unsigned) 0); + } + thr_exit_cleanup(tid, res); + _endthreadex((unsigned) 0); +} + +ethr_tid +ethr_self(void) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + ETHR_ASSERT(0); + return dummy_tid; + } +#endif + /* It is okay for non-ethreads (i.e. native win32 threads) to call + ethr_self(). They will however be returned an invalid tid. */ + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + return dummy_tid; + } + return *tid; +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + /* An invalid tid does not equal any tid, not even an invalid tid */ + return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID; +} + +int +ethr_time_now(ethr_timeval *time) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!time) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + get_curr_time(&time->tv_sec, &time->tv_nsec); + return 0; +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ + DWORD key; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + key = TlsAlloc(); + if (key == TLS_OUT_OF_INDEXES) + return ethr_win_get_errno__(); + *keyp = (ethr_tsd_key) key; + return 0; +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsFree((DWORD) key)) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsSetValue((DWORD) key, (LPVOID) value)) + return ethr_win_get_errno__(); + return 0; +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return (void *) TlsGetValue((DWORD) key); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +ethr_ts_event * +ethr_create_ts_event__(void) +{ + ethr_ts_event *tsep; + ethr_make_ts_event__(&tsep); + return tsep; +} diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index bbc79e9381..0cc315e9be 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -37,21 +37,16 @@ equal_tids/1, mutex/1, try_lock_mutex/1, - recursive_mutex/1, time_now/1, cond_wait/1, - cond_timedwait/1, broadcast/1, detached_thread/1, max_threads/1, - forksafety/1, - vfork/1, tsd/1, spinlock/1, rwspinlock/1, rwmutex/1, - atomic/1, - gate/1]). + atomic/1]). -include_lib("test_server/include/test_server.hrl"). @@ -60,21 +55,16 @@ tests() -> equal_tids, mutex, try_lock_mutex, - recursive_mutex, time_now, cond_wait, - cond_timedwait, broadcast, detached_thread, max_threads, - forksafety, - vfork, tsd, spinlock, rwspinlock, rwmutex, - atomic, - gate]. + atomic]. all(doc) -> []; all(suite) -> tests(). @@ -114,13 +104,6 @@ try_lock_mutex(suite) -> try_lock_mutex(Config) -> run_case(Config, "try_lock_mutex", ""). -recursive_mutex(doc) -> - ["Tests recursive mutexes."]; -recursive_mutex(suite) -> - []; -recursive_mutex(Config) -> - run_case(Config, "recursive_mutex", ""). - time_now(doc) -> ["Tests ethr_time_now by comparing time values with Erlang."]; time_now(suite) -> @@ -169,13 +152,6 @@ cond_wait(suite) -> cond_wait(Config) -> run_case(Config, "cond_wait", ""). -cond_timedwait(doc) -> - ["Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast."]; -cond_timedwait(suite) -> - []; -cond_timedwait(Config) -> - run_case(Config, "cond_timedwait", ""). - broadcast(doc) -> ["Tests that a ethr_cond_broadcast really wakes up all waiting threads"]; broadcast(suite) -> @@ -197,25 +173,6 @@ max_threads(suite) -> max_threads(Config) -> run_case(Config, "max_threads", ""). -forksafety(doc) -> - ["Tests forksafety."]; -forksafety(suite) -> - []; -forksafety(Config) -> - run_case(Config, "forksafety", ""). - -vfork(doc) -> - ["Tests vfork with threads."]; -vfork(suite) -> - case ?t:os_type() of - {unix, osf1} -> - {skip, "vfork() known to hang multi-threaded applications on osf1"}; - _ -> - [] - end; -vfork(Config) -> - run_case(Config, "vfork", ""). - tsd(doc) -> ["Tests thread specific data."]; tsd(suite) -> @@ -251,13 +208,6 @@ atomic(suite) -> atomic(Config) -> run_case(Config, "atomic", ""). -gate(doc) -> - ["Tests gates."]; -gate(suite) -> - []; -gate(Config) -> - run_case(Config, "gate", ""). - %% %% %% Auxiliary functions diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index f779f13c51..7fc71d8047 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -90,10 +90,12 @@ static void print_line(char *frmt,...) print_eol(); } +#if 0 /* Currently not used; silence annoying warning... */ static void print(char *frmt,...) { PRINT_VA_LIST(frmt); } +#endif static void fail(char *frmt,...) { @@ -197,8 +199,8 @@ create_join_thread_test(void) } for (i = 1; i <= CJTT_NO_THREADS; i++) { - int *tres; - res = ethr_thr_join(cjtt_tids[i], (void **) &tres); + void *tres; + res = ethr_thr_join(cjtt_tids[i], &tres); ASSERT(res == 0); ASSERT(tres == &cjtt_res[i]); ASSERT(cjtt_res[i] == i); @@ -216,8 +218,8 @@ create_join_thread_test(void) #define ETT_THREADS 100000 static ethr_tid ett_tids[3]; -static ethr_mutex ett_mutex = ETHR_MUTEX_INITER; -static ethr_cond ett_cond = ETHR_COND_INITER; +static ethr_mutex ett_mutex; +static ethr_cond ett_cond; static int ett_terminate; static void * @@ -234,14 +236,12 @@ static void * ett_thread2(void *unused) { int res; - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); while (!ett_terminate) { res = ethr_cond_wait(&ett_cond, &ett_mutex); ASSERT(res == 0); } - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&ett_mutex); return NULL; } @@ -250,6 +250,10 @@ equal_tids_test(void) { int res, i; + res = ethr_mutex_init(&ett_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&ett_cond); + ASSERT(res == 0); ett_tids[0] = ethr_self(); res = ethr_thr_create(&ett_tids[1], ett_thread, (void *) &ett_tids[1], NULL); @@ -298,13 +302,10 @@ equal_tids_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); ett_terminate = 1; - res = ethr_cond_signal(&ett_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_cond_signal(&ett_cond); + ethr_mutex_unlock(&ett_mutex); res = ethr_thr_join(ett_tids[1], NULL); ASSERT(res == 0); @@ -321,17 +322,14 @@ equal_tids_test(void) * Tests mutexes. */ -static ethr_mutex mt_mutex = ETHR_MUTEX_INITER; +static ethr_mutex mt_mutex; static int mt_data; void * mt_thread(void *unused) { - int res; - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Aux thread locked mutex"); ASSERT(mt_data == 0); @@ -345,8 +343,7 @@ mt_thread(void *unused) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Aux thread unlocked mutex"); return NULL; @@ -356,18 +353,18 @@ mt_thread(void *unused) static void mutex_test(void) { - int do_restart = 1; int res; ethr_tid tid; - print_line("Running test with statically initialized mutex"); + print_line("Trying to initialize mutex"); + res = ethr_mutex_init(&mt_mutex); + ASSERT(res == 0); + print_line("Initialized mutex"); - restart: mt_data = 0; print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 0); @@ -383,8 +380,7 @@ mutex_test(void) ASSERT(mt_data == 0); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); print_line("Main thread goes to sleep for 1 second"); @@ -392,8 +388,7 @@ mutex_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 1); @@ -404,8 +399,7 @@ mutex_test(void) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); res = ethr_thr_join(tid, NULL); @@ -416,20 +410,6 @@ mutex_test(void) ASSERT(res == 0); print_line("Main thread destroyed mutex"); - if (do_restart) { - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_mutex_init(&mt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - - } - } /* @@ -438,9 +418,9 @@ mutex_test(void) * Tests try lock mutex operation. */ -static ethr_mutex tlmt_mtx1 = ETHR_MUTEX_INITER; -static ethr_mutex tlmt_mtx2 = ETHR_MUTEX_INITER; -static ethr_cond tlmt_cnd2 = ETHR_COND_INITER; +static ethr_mutex tlmt_mtx1; +static ethr_mutex tlmt_mtx2; +static ethr_cond tlmt_cnd2; static int tlmt_mtx1_locked; static int tlmt_mtx1_do_unlock; @@ -450,32 +430,24 @@ tlmt_thread(void *unused) { int res; - res = ethr_mutex_lock(&tlmt_mtx1); - ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx1); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (!tlmt_mtx1_do_unlock) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); + ethr_mutex_unlock(&tlmt_mtx1); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 0; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); + ethr_mutex_unlock(&tlmt_mtx2); return NULL; } @@ -486,48 +458,49 @@ try_lock_mutex_test(void) int i, res; ethr_tid tid; + res = ethr_mutex_init(&tlmt_mtx1); + ASSERT(res == 0); + res = ethr_mutex_init(&tlmt_mtx2); + ASSERT(res == 0); + res = ethr_cond_init(&tlmt_cnd2); + ASSERT(res == 0); + tlmt_mtx1_locked = 0; tlmt_mtx1_do_unlock = 0; res = ethr_thr_create(&tid, tlmt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); while (!tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); for (i = 0; i < 10; i++) { res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == EBUSY); } - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_do_unlock = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx1); res = ethr_thr_join(tid, NULL); ASSERT(res == 0); @@ -541,159 +514,6 @@ try_lock_mutex_test(void) } /* - * The recursive mutex test case. - * - * Tests recursive mutexes. - */ - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static ethr_mutex rmt_mutex -#ifdef ETHR_REC_MUTEX_INITER - = ETHR_REC_MUTEX_INITER -#endif - ; -static int rmt_data; - -void * -rmt_thread(void *unused) -{ - int res; - - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread locked mutex"); - - ASSERT(rmt_data == 0); - - rmt_data = 1; - print_line("Aux thread wrote"); - - print_line("Aux thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Aux thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread unlocked mutex"); - - return NULL; -} - -#endif - -static void -recursive_mutex_test(void) -{ -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - int do_restart = 1; - int res; - ethr_tid tid; - -#ifdef ETHR_REC_MUTEX_INITER - print_line("Running test with statically initialized mutex"); -#else - goto dynamic_init; -#endif - - restart: - rmt_data = 0; - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - print_line("Main thread tries to lock mutex again"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex again"); - - ASSERT(rmt_data == 0); - - print_line("Main thread about to create aux thread"); - res = ethr_thr_create(&tid, rmt_thread, NULL, NULL); - ASSERT(res == 0); - print_line("Main thread created aux thread"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex again"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - ASSERT(rmt_data == 1); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - print_line("Main thread joined aux thread"); - - res = ethr_mutex_destroy(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread destroyed mutex"); - - if (do_restart) { -#ifndef ETHR_REC_MUTEX_INITER - dynamic_init: -#endif - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_rec_mutex_init(&rmt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - } - -#ifndef ETHR_REC_MUTEX_INITER - succeed("Static initializer for recursive mutexes not supported"); -#endif - -#else /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - skip("Recursive mutexes not supported"); -#endif /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ -} - -/* * The time now test. * * Tests ethr_time_now by comparing time values with Erlang. @@ -763,106 +583,82 @@ time_now_test(void) */ -static ethr_mutex cwt_mutex = ETHR_MUTEX_INITER; -static ethr_cond cwt_cond = ETHR_COND_INITER; +static ethr_mutex cwt_mutex; +static ethr_cond cwt_cond; static int cwt_counter; void * -cwt_thread(void *is_timedwait_test_ptr) +cwt_thread(void *unused) { - int use_timedwait = *((int *) is_timedwait_test_ptr); int res; - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); - - if (use_timedwait) { - ethr_timeval tv; - res = ethr_time_now(&tv); - ASSERT(res == 0); - tv.tv_sec += 3600; /* Make sure we won't time out */ + ethr_mutex_lock(&cwt_mutex); - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tv); - } while (res == EINTR); - ASSERT(res == 0); - } - else { - do { - res = ethr_cond_wait(&cwt_cond, &cwt_mutex); - } while (res == EINTR); - ASSERT(res == 0); - } + do { + res = ethr_cond_wait(&cwt_cond, &cwt_mutex); + } while (res == EINTR); + ASSERT(res == 0); cwt_counter++; - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); return NULL; } static void -cond_wait_test(int is_timedwait_test) +cond_wait_test(void) { - int do_restart = !is_timedwait_test; ethr_tid tid1, tid2; int res; - if (!is_timedwait_test) - print_line("Running test with statically initialized mutex and cond"); + res = ethr_mutex_init(&cwt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&cwt_cond); + ASSERT(res == 0); - restart: /* Wake with signal */ cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_signal(&cwt_cond); /* Wake one thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake one thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 1); - res = ethr_cond_signal(&cwt_cond); /* Wake the other thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake the other thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 1); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -875,35 +671,30 @@ cond_wait_test(int is_timedwait_test) cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ - ASSERT(res == 0); + ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ do_sleep(1); /* Make sure awakened threads wait on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened threads proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -916,113 +707,6 @@ cond_wait_test(int is_timedwait_test) res = ethr_cond_destroy(&cwt_cond); ASSERT(res == 0); - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } -} - -/* - * The cond timedwait test case. - * - * Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast. - */ - -#define CTWT_MAX_TIME_DIFF 100000 - -static long -ctwt_check_timeout(long to) -{ - int res; - ethr_timeval tva, tvb; - long diff, abs_diff; - - res = ethr_time_now(&tva); - ASSERT(res == 0); - - tva.tv_sec += to / 1000; - tva.tv_nsec += (to % 1000) * 1000000; - if (tva.tv_nsec >= 1000000000) { - tva.tv_sec++; - tva.tv_nsec -= 1000000000; - ASSERT(tva.tv_nsec < 1000000000); - } - - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tva); - } while (res == EINTR); - ASSERT(res == ETIMEDOUT); - - res = ethr_time_now(&tvb); - ASSERT(res == 0); - - diff = (tvb.tv_sec - tva.tv_sec) * 1000000; - diff += (tvb.tv_nsec - tva.tv_nsec)/1000; - - print("Timeout=%ld; ", to); - print("tva.tv_sec=%ld tva.tv_nsec=%ld; ", tva.tv_sec, tva.tv_nsec); - print("tvb.tv_sec=%ld tvb.tv_nsec=%ld; ", tvb.tv_sec, tvb.tv_nsec); - print_line("diff (tvb - tva) = %ld us", diff); - - abs_diff = (long) abs((int) diff); - - ASSERT(CTWT_MAX_TIME_DIFF >= abs_diff); - return abs_diff; -} - -static void -cond_timedwait_test(void) -{ - int do_restart = 1; - long abs_diff, max_abs_diff = 0; - int res; - -#define CTWT_UPD_MAX_DIFF if (abs_diff > max_abs_diff) max_abs_diff = abs_diff; - - print_line("Running test with statically initialized mutex and cond"); - - print_line("CTWT_MAX_TIME_DIFF=%d", CTWT_MAX_TIME_DIFF); - - restart: - - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); - - abs_diff = ctwt_check_timeout(300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(700); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(1000); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(2300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(5100); - CTWT_UPD_MAX_DIFF; - - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); - - cond_wait_test(1); - - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } - - print_line("Max absolute diff = %d us", max_abs_diff); - succeed("Max absolute diff = %d us", max_abs_diff); - -#undef CTWT_UPD_MAX_DIFF } /* @@ -1037,9 +721,9 @@ cond_timedwait_test(void) static int bct_woken = 0; static int bct_waiting = 0; static int bct_done = 0; -static ethr_mutex bct_mutex = ETHR_MUTEX_INITER; -static ethr_cond bct_cond = ETHR_COND_INITER; -static ethr_cond bct_cntrl_cond = ETHR_COND_INITER; +static ethr_mutex bct_mutex; +static ethr_cond bct_cond; +static ethr_cond bct_cntrl_cond; static void * @@ -1047,30 +731,24 @@ bct_thread(void *unused) { int res; - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); while (!bct_done) { bct_waiting++; - if (bct_waiting == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_waiting == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); do { res = ethr_cond_wait(&bct_cond, &bct_mutex); } while (res == EINTR); ASSERT(res == 0); bct_woken++; - if (bct_woken == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_woken == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); } - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); return NULL; } @@ -1081,14 +759,20 @@ broadcast_test(void) int res, i; ethr_tid tid[BCT_THREADS]; + res = ethr_mutex_init(&bct_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cntrl_cond); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cond); + ASSERT(res == 0); + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_create(&tid[i], bct_thread, NULL, NULL); ASSERT(res == 0); } - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); for (i = 0; i < BCT_NO_OF_WAITS; i++) { @@ -1101,8 +785,7 @@ broadcast_test(void) bct_woken = 0; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); while (bct_woken != BCT_THREADS) { res = ethr_cond_wait(&bct_cntrl_cond, &bct_mutex); @@ -1114,13 +797,11 @@ broadcast_test(void) bct_done = 1; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); - for (i = 0; i < BCT_THREADS - 1; i++) { + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_join(tid[i], NULL); ASSERT(res == 0); } @@ -1143,26 +824,22 @@ broadcast_test(void) #define DT_THREADS (50*1024) #define DT_BATCH_SIZE 64 -static ethr_mutex dt_mutex = ETHR_MUTEX_INITER; -static ethr_cond dt_cond = ETHR_COND_INITER; +static ethr_mutex dt_mutex; +static ethr_cond dt_cond; static int dt_count; static int dt_limit; static void * dt_thread(void *unused) { - int res; - - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); dt_count++; if (dt_count >= dt_limit) ethr_cond_signal(&dt_cond); - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); return NULL; } @@ -1174,6 +851,11 @@ detached_thread_test(void) ethr_tid tid[DT_BATCH_SIZE]; int i, j, res; + res = ethr_mutex_init(&dt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&dt_cond); + ASSERT(res == 0); + thr_opts.detached = 1; dt_count = 0; dt_limit = 0; @@ -1187,14 +869,12 @@ detached_thread_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); while (dt_count < dt_limit) { res = ethr_cond_wait(&dt_cond, &dt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); print_line("dt_count = %d", dt_count); } @@ -1211,8 +891,8 @@ detached_thread_test(void) #define MTT_TIMES 10 static int mtt_terminate; -static ethr_mutex mtt_mutex = ETHR_MUTEX_INITER; -static ethr_cond mtt_cond = ETHR_COND_INITER; +static ethr_mutex mtt_mutex; +static ethr_cond mtt_cond; static char mtt_string[22*MTT_TIMES]; /* 22 is enough for ", %d" */ @@ -1220,16 +900,14 @@ void *mtt_thread(void *unused) { int res; - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); while (!mtt_terminate) { res = ethr_cond_wait(&mtt_cond, &mtt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); return NULL; } @@ -1265,16 +943,13 @@ mtt_create_join_threads(void) print_line("%d = ethr_thr_create()", res); print_line("Number of created threads: %d", no_threads); - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); mtt_terminate = 1; - res = ethr_cond_broadcast(&mtt_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&mtt_cond); - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); while (ix) { res = ethr_thr_join(tids[--ix], NULL); @@ -1292,9 +967,14 @@ mtt_create_join_threads(void) static void max_threads_test(void) { - int no_threads[MTT_TIMES], i, up, down, eq; + int no_threads[MTT_TIMES], i, up, down, eq, res; char *str; + res = ethr_mutex_init(&mtt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&mtt_cond); + ASSERT(res == 0); + for (i = 0; i < MTT_TIMES; i++) { no_threads[i] = mtt_create_join_threads(); } @@ -1326,272 +1006,6 @@ max_threads_test(void) } - -/* - * The forksafety test case. - * - * Tests forksafety. - */ -#ifdef __WIN32__ -#define NO_FORK_PRESENT -#endif - -#ifndef NO_FORK_PRESENT - -static ethr_mutex ft_test_inner_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_test_outer_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_go_mutex = ETHR_MUTEX_INITER; -static ethr_cond ft_go_cond = ETHR_COND_INITER; -static int ft_go; -static int ft_have_forked; - -static void * -ft_thread(void *unused) -{ - int res; - - res = ethr_mutex_lock(&ft_test_outer_mutex); - ASSERT(res == 0); - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - ft_go = 1; - res = ethr_cond_signal(&ft_go_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_lock(&ft_test_inner_mutex); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_test_inner_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_unlock(&ft_test_outer_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(ft_have_forked); - - - return NULL; -} - -#endif /* #ifndef NO_FORK_PRESENT */ - -static void -forksafety_test(void) -{ -#ifdef NO_FORK_PRESENT - skip("No fork() present; nothing to test"); -#elif defined(DEBUG) - skip("Doesn't work in debug build"); -#else - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - ethr_tid tid; - int res; - int fds[2]; - - - res = ethr_mutex_set_forksafe(&ft_test_inner_mutex); - if (res == ENOTSUP) { - skip("Forksafety not supported on this platform!"); - } - ASSERT(res == 0); - res = ethr_mutex_set_forksafe(&ft_test_outer_mutex); - ASSERT(res == 0); - - - res = pipe(fds); - ASSERT(res == 0); - - ft_go = 0; - ft_have_forked = 0; - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, ft_thread, NULL, NULL); - ASSERT(res == 0); - - do { - res = ethr_cond_wait(&ft_go_cond, &ft_go_mutex); - } while (res == EINTR || !ft_go); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - res = fork(); - ft_have_forked = 1; - if (res == 0) { - close(fds[0]); - res = ethr_mutex_lock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_lock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = ethr_mutex_destroy(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_destroy(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = (int) write(fds[1], (void *) snd_msg, sizeof(snd_msg)); - if (res != sizeof(snd_msg)) - _exit(1); - close(fds[1]); - _exit(0); - } - ASSERT(res > 0); - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); -#endif -} - - -/* - * The vfork test case. - * - * Tests vfork with threads. - */ - -#ifdef __WIN32__ -#define NO_VFORK_PRESENT -#endif - -#ifndef NO_VFORK_PRESENT - -#undef vfork - -static ethr_mutex vt_mutex = ETHR_MUTEX_INITER; - -static void * -vt_thread(void *vprog) -{ - char *prog = (char *) vprog; - int res; - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - int fds[2]; - char closefd[20]; - char writefd[20]; - - res = pipe(fds); - ASSERT(res == 0); - - res = sprintf(closefd, "%d", fds[0]); - ASSERT(res <= 20); - res = sprintf(writefd, "%d", fds[1]); - ASSERT(res <= 20); - - print("parent: About to vfork and execute "); - print("execlp(\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", NULL)", - prog, prog, "vfork", "exec", snd_msg, closefd, writefd); - print_line(" in child"); - res = vfork(); - if (res == 0) { - execlp(prog, prog, "vfork", "exec", snd_msg, closefd, writefd, NULL); - _exit(1); - } - ASSERT(res > 0); - - print_line("parent: I'm back"); - - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - print_line("parent: %d = read()", res); - print_line("parent: rec_msg=\"%s\"", rec_msg); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); - - return NULL; -} - -#endif /* #ifndef NO_VFORK_PRESENT */ - -static void -vfork_test(int argc, char *argv[]) -{ -#ifdef NO_VFORK_PRESENT - skip("No vfork() present; nothing to test"); -#else - int res; - ethr_tid tid; - - if (argc == 6 && strcmp("exec", argv[2]) == 0) { - /* We are child after vfork() and execlp() ... */ - - char *snd_msg; - int closefd; - int writefd; - - snd_msg = argv[3]; - closefd = atoi(argv[4]); - writefd = atoi(argv[5]); - - print_line("child: snd_msg=\"%s\"; closefd=%d writefd=%d", - snd_msg, closefd, writefd); - - close(closefd); - - res = (int) write(writefd, (void *) snd_msg, strlen(snd_msg)+1); - print_line("child: %d = write()", res); - if (res != strlen(snd_msg)+1) - exit(1); - close(writefd); - print_line("child: bye"); - exit(0); - } - ASSERT(argc == 2); - - res = ethr_mutex_set_forksafe(&vt_mutex); - ASSERT(res == 0 || res == ENOTSUP); - res = ethr_mutex_lock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, vt_thread, (void *) argv[0], NULL); - ASSERT(res == 0); - - do_sleep(1); - - res = ethr_mutex_unlock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - - res = ethr_mutex_destroy(&vt_mutex); - ASSERT(res == 0); -#endif -} - - /* * The tsd test case. * @@ -1651,11 +1065,8 @@ static int st_data; void * st_thread(void *unused) { - int res; - print_line("Aux thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Aux thread locked spinlock"); ASSERT(st_data == 0); @@ -1669,8 +1080,7 @@ st_thread(void *unused) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Aux thread unlocked spinlock"); return NULL; @@ -1691,8 +1101,7 @@ spinlock_test(void) st_data = 0; print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 0); @@ -1708,8 +1117,7 @@ spinlock_test(void) ASSERT(st_data == 0); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); print_line("Main thread goes to sleep for 1 second"); @@ -1717,8 +1125,7 @@ spinlock_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 1); @@ -1729,8 +1136,7 @@ spinlock_test(void) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); res = ethr_thr_join(tid, NULL); @@ -1757,23 +1163,19 @@ void * rwst_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Aux thread read locked rwspinlock"); ASSERT(rwst_data == 4711); print_line("Aux thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Aux thread read unlocked rwspinlock"); print_line("Aux thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Aux thread write locked rwspinlock"); data = ++rwst_data; @@ -1787,8 +1189,7 @@ rwst_thread(void *unused) ++rwst_data; print_line("Aux thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Aux thread write unlocked rwspinlock"); return NULL; @@ -1810,8 +1211,7 @@ rwspinlock_test(void) rwst_data = 4711; print_line("Main thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Main thread read locked rwspinlock"); ASSERT(rwst_data == 4711); @@ -1828,13 +1228,11 @@ rwspinlock_test(void) ASSERT(rwst_data == 4711); print_line("Main thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Main thread read unlocked rwspinlock"); print_line("Main thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Main thread write locked rwspinlock"); data = ++rwst_data; @@ -1847,8 +1245,7 @@ rwspinlock_test(void) ++rwst_data; print_line("Main thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Main thread write unlocked rwspinlock"); res = ethr_thr_join(tid, NULL); @@ -1875,23 +1272,19 @@ void * rwmt_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Aux thread read locked rwmutex"); ASSERT(rwmt_data == 4711); print_line("Aux thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Aux thread read unlocked rwmutex"); print_line("Aux thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Aux thread write locked rwmutex"); data = ++rwmt_data; @@ -1905,8 +1298,7 @@ rwmt_thread(void *unused) ++rwmt_data; print_line("Aux thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Aux thread write unlocked rwmutex"); return NULL; @@ -1928,8 +1320,7 @@ rwmutex_test(void) rwmt_data = 4711; print_line("Main thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Main thread read locked rwmutex"); ASSERT(rwmt_data == 4711); @@ -1946,13 +1337,11 @@ rwmutex_test(void) ASSERT(rwmt_data == 4711); print_line("Main thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Main thread read unlocked rwmutex"); print_line("Main thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Main thread write locked rwmutex"); data = ++rwmt_data; @@ -1965,8 +1354,7 @@ rwmutex_test(void) ++rwmt_data; print_line("Main thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Main thread write unlocked rwmutex"); res = ethr_thr_join(tid, NULL); @@ -1998,65 +1386,53 @@ static ethr_atomic_t at_data; void * at_thread(void *unused) { - int res, i; + int i; long val, go; - res = ethr_atomic_inctest(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_ready); ASSERT(val > 0); ASSERT(val <= AT_THREADS); do { - res = ethr_atomic_read(&at_go, &go); - ASSERT(res == 0); + go = ethr_atomic_read(&at_go); } while (!go); for (i = 0; i < AT_ITER; i++) { - res = ethr_atomic_or_old(&at_data, at_set_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_bor(&at_data, at_set_val); ASSERT(val >= (i == 0 ? 0 : at_set_val) + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_and_old(&at_data, ~at_rm_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_band(&at_data, ~at_rm_val); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inctest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_data); ASSERT(val > at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_dectest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_dec_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inc(&at_data); - ASSERT(res == 0); + ethr_atomic_inc(&at_data); - res = ethr_atomic_dec(&at_data); - ASSERT(res == 0); + ethr_atomic_dec(&at_data); - res = ethr_atomic_addtest(&at_data, (long) 4711, &val); - ASSERT(res == 0); + val = ethr_atomic_add_read(&at_data, (long) 4711); ASSERT(val >= at_set_val + (long) 2*4711); ASSERT(val <= at_max_val); - res = ethr_atomic_add(&at_data, (long) -4711); - ASSERT(res == 0); + ethr_atomic_add(&at_data, (long) -4711); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); } - res = ethr_atomic_inc(&at_done); - ASSERT(res == 0); + ethr_atomic_inc(&at_done); return NULL; } @@ -2069,14 +1445,13 @@ atomic_test(void) ethr_tid tid[AT_THREADS]; ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; - if (sizeof(long) > 4) { +#if ETHR_SIZEOF_PTR > 4 at_rm_val = ((long) 1) << 57; at_set_val = ((long) 1) << 60; - } - else { +#else at_rm_val = ((long) 1) << 27; at_set_val = ((long) 1) << 30; - } +#endif at_max_val = at_set_val + at_rm_val + ((long) AT_THREADS + 1) * 4711; data_init = at_rm_val + (long) 4711; @@ -2085,22 +1460,15 @@ atomic_test(void) thr_opts.detached = 1; print_line("Initializing"); - res = ethr_atomic_init(&at_ready, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_go, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_done, data_init); - ASSERT(res == 0); - res = ethr_atomic_init(&at_data, data_init); - ASSERT(res == 0); + ethr_atomic_init(&at_ready, 0); + ethr_atomic_init(&at_go, 0); + ethr_atomic_init(&at_done, data_init); + ethr_atomic_init(&at_data, data_init); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val == data_init); - res = ethr_atomic_set(&at_done, 0); - ASSERT(res == 0); - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + ethr_atomic_set(&at_done, 0); + val = ethr_atomic_read(&at_done); ASSERT(val == 0); print_line("Creating threads"); @@ -2111,31 +1479,27 @@ atomic_test(void) print_line("Waiting for threads to ready up"); do { - res = ethr_atomic_read(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_ready); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Letting threads loose"); - res = ethr_atomic_xchg(&at_go, 17, &val); - ASSERT(res == 0); + val = ethr_atomic_xchg(&at_go, 17); ASSERT(val == 0); - res = ethr_atomic_read(&at_go, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_go); ASSERT(val == 17); print_line("Waiting for threads to finish"); do { - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_done); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Checking result"); - res = ethr_atomic_read(&at_data, &val); + val = ethr_atomic_read(&at_data); ASSERT(res == 0); ASSERT(val == data_final); print_line("Result ok"); @@ -2143,190 +1507,6 @@ atomic_test(void) } -/* - * The gate test case. - * - * Tests gates. - */ - -#define GT_THREADS 10 - -static ethr_atomic_t gt_wait1; -static ethr_atomic_t gt_wait2; -static ethr_atomic_t gt_done; - -static ethr_gate gt_gate1; -static ethr_gate gt_gate2; - -void * -gt_thread(void *thr_no) -{ - int no = (int)(long) thr_no; - int swait = no % 2 == 0; - int res; - long done; - - - do { - - res = ethr_atomic_inc(>_wait1); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate1, INT_MAX); - else - res = ethr_gate_wait(>_gate1); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait1); - ASSERT(res == 0); - - res = ethr_atomic_inc(>_wait2); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate2, INT_MAX); - else - res = ethr_gate_wait(>_gate2); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait2); - ASSERT(res == 0); - - res = ethr_atomic_read(>_done, &done); - ASSERT(res == 0); - } while (!done); - return NULL; -} - - -static void -gate_test(void) -{ - long val; - int res, i; - ethr_tid tid[GT_THREADS]; - - print_line("Initializing"); - res = ethr_atomic_init(>_wait1, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_wait2, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_done, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate2); - ASSERT_EQ(res, 0, "%d"); - - print_line("Creating threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_create(&tid[i], gt_thread, (void *) i, NULL); - ASSERT_EQ(res, 0, "%d"); - } - - print_line("Waiting for threads to ready up"); - do { - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT(0 <= val && val <= GT_THREADS); - } while (val != GT_THREADS); - - print_line("Testing"); - - res = ethr_gate_let_through(>_gate1, 8); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 8)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 8, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 8, "%ld"); - - res = ethr_gate_let_through(>_gate2, 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_gate_let_through(>_gate1, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == GT_THREADS)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS, "%ld"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_atomic_set(>_done, 1); - ASSERT_EQ(res, 0, "%d"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_let_through(>_gate1, GT_THREADS - 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM(((res = ethr_atomic_read(>_wait1, &val)) != 0 - || (val == 0 - && ((res = ethr_atomic_read(>_wait2, &val)) != 0 - || val == 0))), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - print_line("Joining threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_join(tid[i], NULL); - ASSERT_EQ(res, 0, "%d"); - } - - res = ethr_gate_destroy(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_destroy(>_gate2); - ASSERT_EQ(res, 0, "%d"); - -} - #endif /* #ifndef ETHR_NO_THREAD_LIB */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -2342,14 +1522,12 @@ main(int argc, char *argv[]) #ifndef ETHR_NO_THREAD_LIB { char *testcase; - int res; send_my_pid(); testcase = argv[1]; - res = ethr_init(NULL); - if (res != 0) + if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0) fail("Failed to initialize the ethread library"); if (strcmp(testcase, "create_join_thread") == 0) @@ -2360,24 +1538,16 @@ main(int argc, char *argv[]) mutex_test(); else if (strcmp(testcase, "try_lock_mutex") == 0) try_lock_mutex_test(); - else if (strcmp(testcase, "recursive_mutex") == 0) - recursive_mutex_test(); else if (strcmp(testcase, "time_now") == 0) time_now_test(); else if (strcmp(testcase, "cond_wait") == 0) - cond_wait_test(0); - else if (strcmp(testcase, "cond_timedwait") == 0) - cond_timedwait_test(); + cond_wait_test(); else if (strcmp(testcase, "broadcast") == 0) broadcast_test(); else if (strcmp(testcase, "detached_thread") == 0) detached_thread_test(); else if (strcmp(testcase, "max_threads") == 0) max_threads_test(); - else if (strcmp(testcase, "forksafety") == 0) - forksafety_test(); - else if (strcmp(testcase, "vfork") == 0) - vfork_test(argc, argv); else if (strcmp(testcase, "tsd") == 0) tsd_test(); else if (strcmp(testcase, "spinlock") == 0) @@ -2388,8 +1558,6 @@ main(int argc, char *argv[]) rwmutex_test(); else if (strcmp(testcase, "atomic") == 0) atomic_test(); - else if (strcmp(testcase, "gate") == 0) - gate_test(); else skip("Test case \"%s\" not implemented yet", testcase); |