diff options
| author | Rickard Green <[email protected]> | 2010-06-17 10:23:50 +0200 |
|---|---|---|
| committer | Rickard Green <[email protected]> | 2010-08-10 11:41:14 +0200 |
| commit | 300b419486c1ca88e33938f182d5d5a8b90fb73f (patch) | |
| tree | 55c37d5fb042bf6b3b5f56d89c9238a7e22f8b29 | |
| parent | c1e94fa9a6fe4ae717d35dfbd1b628dc2e06d26a (diff) | |
| download | otp-300b419486c1ca88e33938f182d5d5a8b90fb73f.tar.gz otp-300b419486c1ca88e33938f182d5d5a8b90fb73f.tar.bz2 otp-300b419486c1ca88e33938f182d5d5a8b90fb73f.zip | |
Rewrite ethread library
Large parts of the ethread library have been rewritten. The
ethread library is an Erlang runtime system internal, portable
thread library used by the runtime system itself.
Most notable improvement is a reader optimized rwlock
implementation which dramatically improve the performance of
read-lock/read-unlock operations on multi processor systems by
avoiding ping-ponging of the rwlock cache lines. The reader
optimized rwlock implementation is used by miscellaneous
rwlocks in the runtime system that are known to be read-locked
frequently, and can be enabled on ETS tables by passing the
`{read_concurrency, true}' option upon table creation. See the
documentation of `ets:new/2' for more information.
The ethread library can now also use the libatomic_ops library
for atomic memory accesses. This makes it possible for the
Erlang runtime system to utilize optimized atomic operations
on more platforms than before. Use the
`--with-libatomic_ops=PATH' configure command line argument
when specifying where the libatomic_ops installation is
located. The libatomic_ops library can be downloaded from:
http://www.hpl.hp.com/research/linux/atomic_ops/
The changed API of the ethread library has also caused
modifications in the Erlang runtime system. Preparations for
the to come "delayed deallocation" feature has also been done
since it depends on the ethread library.
Note: When building for x86, the ethread library will now use
instructions that first appeared on the pentium 4 processor. If
you want the runtime system to be compatible with older
processors (back to 486) you need to pass the
`--enable-ethread-pre-pentium4-compatibility' configure command
line argument when configuring the system.
77 files changed, 11355 insertions, 6872 deletions
diff --git a/INSTALL.md b/INSTALL.md index b26c0ce018..2567b791e5 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -273,6 +273,16 @@ Some of the available `configure` options are: * `--with-ssl=PATH` - Specify location of OpenSSL include and lib * `--{with,without}-ssl` - OpenSSL (without implies that the `crypto`, `ssh`, and `ssl` won't be built) +* `--enable-ethread-pre-pentium4-compatibility` - Enable compatibility with + x86 processors before pentium 4 (back to 486) in the ethread library. If + not passed the ethread library (part of the runtime system) will use + instructions that first appeared on the pentium 4 processor when building + for x86. +* `--with-libatomic_ops=PATH` - Use the `libatomic_ops` library for atomic + memory accesses. If `configure` should inform you about no native atomic + implementation available, you typically want to try using the + `libatomic_ops` library. It can be downloaded from + <http://www.hpl.hp.com/research/linux/atomic_ops/>. If you or your system has special requirements please read the `Makefile` for additional configuration information. diff --git a/configure.in b/configure.in index eb29b13bcc..f9fa34cedf 100644 --- a/configure.in +++ b/configure.in @@ -368,11 +368,12 @@ if test "$files" != "$pattern"; then fi pattern="lib/*/CONF_INFO" files=`echo $pattern` -if test "$files" != "$pattern"; then +if test "$files" != "$pattern" || test -f erts/CONF_INFO; then echo '*********************************************************************' echo '********************** APPLICATIONS INFORMATION *******************' echo '*********************************************************************' echo + test ! -f erts/CONF_INFO || files="$files erts/CONF_INFO" for infofile in $files; do app=`dirname $infofile`; app=`basename $app` printf "%-15s: " $app; cat $infofile diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 61244c7cd3..0fa3fec244 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -653,6 +653,19 @@ fi ]) +AC_DEFUN(ERL_INTERNAL_LIBS, +[ + +ERTS_INTERNAL_X_LIBS= + +AC_CHECK_LIB(kstat, kstat_open, +[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) +ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) + +AC_SUBST(ERTS_INTERNAL_X_LIBS) + +]) + dnl ---------------------------------------------------------------------- dnl dnl ERL_FIND_ETHR_LIB @@ -676,10 +689,13 @@ AC_DEFUN(ERL_FIND_ETHR_LIB, [ LM_CHECK_THR_LIB +ERL_INTERNAL_LIBS +ethr_have_native_atomics=no +ethr_have_native_spinlock=no ETHR_THR_LIB_BASE="$THR_LIB_NAME" ETHR_DEFS="$THR_DEFS" -ETHR_X_LIBS="$THR_LIBS" +ETHR_X_LIBS="$THR_LIBS $ERTS_INTERNAL_X_LIBS" ETHR_LIBS= ETHR_LIB_NAME= @@ -691,6 +707,7 @@ ethr_lib_name=ethread case "$THR_LIB_NAME" in win32_threads) + ETHR_THR_LIB_BASE_DIR=win # * _WIN32_WINNT >= 0x0400 is needed for # TryEnterCriticalSection # * _WIN32_WINNT >= 0x0403 is needed for @@ -716,10 +733,13 @@ case "$THR_LIB_NAME" in if test $found_win32_winnt = no; then AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS]) fi + ethr_have_native_atomics=yes + ethr_have_native_spinlock=yes AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) ;; pthread) + ETHR_THR_LIB_BASE_DIR=pthread AC_DEFINE(ETHR_PTHREADS, 1, [Define if you have pthreads]) case $host_os in openbsd*) @@ -775,9 +795,7 @@ case "$THR_LIB_NAME" in if test $usable_sigaltstack = no; then ETHR_DEFS="$ETHR_DEFS -DETHR_UNUSABLE_SIGALTSTACK" fi - - AC_DEFINE(ETHR_INIT_MUTEX_IN_CHILD_AT_FORK, 1, \ -[Define if mutexes should be reinitialized (instead of unlocked) in child at fork.]) ;; + ;; *) ;; esac @@ -808,6 +826,10 @@ case "$THR_LIB_NAME" in [Define if you need the <nptl/pthread.h> header file.]) fi + AC_CHECK_HEADER(sched.h, \ + AC_DEFINE(ETHR_HAVE_SCHED_H, 1, \ +[Define if you have the <sched.h> header file.])) + AC_CHECK_HEADER(sys/time.h, \ AC_DEFINE(ETHR_HAVE_SYS_TIME_H, 1, \ [Define if you have the <sys/time.h> header file.])) @@ -823,26 +845,63 @@ case "$THR_LIB_NAME" in dnl Check for functions dnl - AC_CHECK_FUNC(pthread_atfork, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_ATFORK, 1, \ -[Define if you have the pthread_atfork function.])) - AC_CHECK_FUNC(pthread_mutexattr_settype, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE, 1, \ -[Define if you have the pthread_mutexattr_settype function.])) - AC_CHECK_FUNC(pthread_mutexattr_setkind_np, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP, 1, \ -[Define if you have the pthread_mutexattr_setkind_np function.])) AC_CHECK_FUNC(pthread_spin_lock, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ -[Define if you have the pthread_spin_lock function.])) + [ethr_have_native_spinlock=yes \ + AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ +[Define if you have the pthread_spin_lock function.])]) + + have_sched_yield=no + have_librt_sched_yield=no + AC_CHECK_FUNC(sched_yield, [have_sched_yield=yes]) + if test $have_sched_yield = no; then + AC_CHECK_LIB(rt, sched_yield, + [have_librt_sched_yield=yes + ETHR_X_LIBS="$ETHR_X_LIBS -lrt"]) + fi + if test $have_sched_yield = yes || test $have_librt_sched_yield = yes; then + AC_DEFINE(ETHR_HAVE_SCHED_YIELD, 1, [Define if you have the sched_yield() function.]) + AC_MSG_CHECKING([whether sched_yield() returns an int]) + sched_yield_ret_int=no + AC_TRY_COMPILE([ + #ifdef ETHR_HAVE_SCHED_H + #include <sched.h> + #endif + ], + [int sched_yield();], + [sched_yield_ret_int=yes]) + AC_MSG_RESULT([$sched_yield_ret_int]) + if test $sched_yield_ret_int = yes; then + AC_DEFINE(ETHR_SCHED_YIELD_RET_INT, 1, [Define if sched_yield() returns an int.]) + fi + fi + + have_pthread_yield=no + AC_CHECK_FUNC(pthread_yield, [have_pthread_yield=yes]) + if test $have_pthread_yield = yes; then + AC_DEFINE(ETHR_HAVE_PTHREAD_YIELD, 1, [Define if you have the pthread_yield() function.]) + AC_MSG_CHECKING([whether pthread_yield() returns an int]) + pthread_yield_ret_int=no + AC_TRY_COMPILE([ + #if defined(ETHR_NEED_NPTL_PTHREAD_H) + #include <nptl/pthread.h> + #elif defined(ETHR_HAVE_MIT_PTHREAD_H) + #include <pthread/mit/pthread.h> + #elif defined(ETHR_HAVE_PTHREAD_H) + #include <pthread.h> + #endif + ], + [int pthread_yield();], + [pthread_yield_ret_int=yes]) + AC_MSG_RESULT([$pthread_yield_ret_int]) + if test $pthread_yield_ret_int = yes; then + AC_DEFINE(ETHR_PTHREAD_YIELD_RET_INT, 1, [Define if pthread_yield() returns an int.]) + fi + fi have_pthread_rwlock_init=no AC_CHECK_FUNC(pthread_rwlock_init, [have_pthread_rwlock_init=yes]) if test $have_pthread_rwlock_init = yes; then - AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_INIT, 1, \ -[Define if you have a pthread_rwlock implementation that can be used.]) - ethr_have_pthread_rwlockattr_setkind_np=no AC_CHECK_FUNC(pthread_rwlockattr_setkind_np, [ethr_have_pthread_rwlockattr_setkind_np=yes]) @@ -876,12 +935,42 @@ case "$THR_LIB_NAME" in fi fi + if test "$force_pthread_rwlocks" = "yes"; then + AC_DEFINE(ETHR_FORCE_PTHREAD_RWLOCK, 1, \ +[Define if you want to force usage of pthread rwlocks]) + + if test $have_pthread_rwlock_init = yes; then + AC_MSG_WARN([Forced usage of pthread rwlocks. Note that this implementation may suffer from starvation issues.]) + else + AC_MSG_ERROR([User forced usage of pthread rwlock, but no such implementation was found]) + fi + fi AC_CHECK_FUNC(pthread_attr_setguardsize, \ AC_DEFINE(ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE, 1, \ [Define if you have the pthread_attr_setguardsize function.])) + linux_futex=no + AC_MSG_CHECKING([for Linux futexes]) + AC_TRY_LINK([ + #include <sys/syscall.h> + #include <unistd.h> + #include <linux/futex.h> + #include <sys/time.h> + ], + [ + int i = 1; + syscall(__NR_futex, (void *) &i, FUTEX_WAKE, 1, + (void*)0,(void*)0, 0); + syscall(__NR_futex, (void *) &i, FUTEX_WAIT, 0, + (void*)0,(void*)0, 0); + return 0; + ], + linux_futex=yes) + AC_MSG_RESULT([$linux_futex]) + test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.]) + AC_MSG_CHECKING([for GCC atomic operations]) ethr_have_gcc_atomic_ops=no AC_TRY_LINK([], @@ -899,10 +988,69 @@ case "$THR_LIB_NAME" in AC_MSG_RESULT([$ethr_have_gcc_atomic_ops]) test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations]) + case "$host_cpu" in + sun4u | sparc64 | sun4v) + ethr_have_native_atomics=yes;; + i86pc | i386 | i486 | i586 | i686 | x86_64 | amd64) + ethr_have_native_atomics=yes;; + macppc | ppc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac + + AC_MSG_CHECKING([for a usable libatomic_ops implementation]) + case "x$with_libatomic_ops" in + xno | xyes | x) + libatomic_ops_include= + ;; + *) + if test -d "${with_libatomic_ops}/include"; then + libatomic_ops_include="-I$with_libatomic_ops/include" + CPPFLAGS="$CPPFLAGS $libatomic_ops_include" + else + AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found]) + fi;; + esac + ethr_have_libatomic_ops=no + AC_TRY_LINK([#include "atomic_ops.h"], + [ + volatile AO_t x; + AO_t y; + int z; + + AO_nop_full(); + AO_store(&x, (AO_t) 0); + z = AO_load(&x); + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); + ], + [ethr_have_native_atomics=yes + ethr_have_libatomic_ops=yes]) + AC_MSG_RESULT([$ethr_have_libatomic_ops]) + if test $ethr_have_libatomic_ops = yes; then + AC_CHECK_SIZEOF(AO_t, , + [ + #include <stdio.h> + #include "atomic_ops.h" + ]) + AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used]) + + AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations]) + if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations]) + fi + ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include" + elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_MSG_ERROR([No usable libatomic_ops implementation found]) + fi + dnl Restore LIBS LIBS=$saved_libs dnl restore CPPFLAGS CPPFLAGS=$saved_cppflags + ;; *) ;; @@ -918,16 +1066,23 @@ fi if test "x$ETHR_THR_LIB_BASE" != "x"; then ETHR_DEFS="-DUSE_THREADS $ETHR_DEFS" - ETHR_LIBS="-l$ethr_lib_name $ETHR_X_LIBS" + ETHR_LIBS="-l$ethr_lib_name -lerts_internal_r $ETHR_X_LIBS" ETHR_LIB_NAME=$ethr_lib_name fi AC_CHECK_SIZEOF(void *) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers]) -if test "X$disable_native_ethr_impls" = "Xyes"; then - AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) -fi +AC_ARG_ENABLE(native-ethr-impls, + AS_HELP_STRING([--disable-native-ethr-impls], + [disable native ethread implementations]), +[ case "$enableval" in + no) disable_native_ethr_impls=yes ;; + *) disable_native_ethr_impls=no ;; + esac ], disable_native_ethr_impls=no) + +test "X$disable_native_ethr_impls" = "Xyes" && + AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], @@ -940,6 +1095,21 @@ AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, test $enable_prefer_gcc_native_ethr_impls = yes && AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) +AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, + AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], + [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), +[ case "$enableval" in + yes) enable_ethread_pre_pentium4_compatibility=yes ;; + *) enable_ethread_pre_pentium4_compatibilit=no ;; + esac ], enable_ethread_pre_pentium4_compatibilit=no) + +test $enable_ethread_pre_pentium4_compatibilit = yes && + AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.]) + +AC_ARG_WITH(libatomic_ops, + AS_HELP_STRING([--with-libatomic_ops=PATH], + [use libatomic_ops with the ethread library])) + AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) @@ -948,6 +1118,7 @@ AC_SUBST(ETHR_LIBS) AC_SUBST(ETHR_LIB_NAME) AC_SUBST(ETHR_DEFS) AC_SUBST(ETHR_THR_LIB_BASE) +AC_SUBST(ETHR_THR_LIB_BASE_DIR) ]) diff --git a/erts/configure.in b/erts/configure.in index 63bf548c89..157163fe0b 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -61,6 +61,9 @@ if test x"${ERL_TOP}/erts" != x"$srcdir"; then fi erl_top=${ERL_TOP} +# Remove old configuration information +/bin/rm -f "$ERL_TOP/erts/CONF_INFO" + # echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX # echo X # echo "X srcdir = $srcdir" @@ -277,20 +280,6 @@ AC_ARG_ENABLE(clock-gettime, *) clock_gettime_correction=yes ;; esac ], clock_gettime_correction=unknown) -AC_ARG_ENABLE(native-ethr-impls, -[ --enable-native-ethr-impls enable native ethread implementations - --disable-native-ethr-impls disable native ethread implementations], -[ case "$enableval" in - no) disable_native_ethr_impls=yes ;; - *) disable_native_ethr_impls=no ;; - esac ], disable_native_ethr_impls=no) - -dnl Defined in libraries/megaco/configure.in but we need it here -dnl also in order to show it to the "top user" - -AC_ARG_ENABLE(megaco_flex_scanner_lineno, -[ --disable-megaco-flex-scanner-lineno disable megaco flex scanner lineno]) - dnl Magic test for clearcase. OTP_RELEASE= if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then @@ -1049,6 +1038,42 @@ if test $ERTS_BUILD_SMP_EMU = yes; then fi AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built]) + + case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in + yes-*) + ;; + + no-yes) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + No native atomic implementation available. + Fallbacks implemented using spinlocks will be + used. Note that the performance of the SMP + runtime system will suffer due to this. + +EOF + ;; + + no-no) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF + + No native atomic implementation, nor no native + spinlock implementation available. Fallbacks + implemented using mutexes will be used. Note + that the performance of the SMP runtime system + will suffer much due to this. + +EOF + ;; + + esac + enable_threads=force fi @@ -1221,13 +1246,7 @@ fi AC_SUBST(EMU_LOCK_CHECKING) -ERTS_INTERNAL_X_LIBS= - -AC_CHECK_LIB(kstat, kstat_open, -[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) -ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) - -AC_SUBST(ERTS_INTERNAL_X_LIBS) +ERL_INTERNAL_LIBS dnl THR_LIBS and THR_DEFS are only used by odbc THR_LIBS=$ETHR_X_LIBS diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index df80142ce1..0e26d62548 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -603,6 +603,24 @@ <item> <p>Force ets memory block to be moved on realloc.</p> </item> + <tag><marker id="+rg"><c><![CDATA[+rg ReaderGroupsLimit]]></c></marker></tag> + <item> + <p>Limits the amount of reader groups used by read/write locks + optimized for read operations in the Erlang runtime system. By + default the reader groups limit equals 8.</p> + <p>When the amount of schedulers is less than or equal to the reader + groups limit, each scheduler has its own reader group. When the + amount of schedulers is larger than the reader groups limit, + schedulers share reader groups. Shared reader groups degrades + read lock and read unlock performance while a large amount of + reader groups degrades write lock performance, so the limit is a + tradeoff between performance for read operations and performance + for write operations. Each reader group currently consumes 64 byte + in each read/write lock. Also note that a runtime system using + shared reader groups benefits from <seealso marker="#+sbt">binding + schedulers to logical processors</seealso>, since the reader groups + are distributed better between schedulers.</p> + </item> <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag> <item> <p>Sets the amount of scheduler threads to create and scheduler diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index eca6121a1e..2b9f70b0f4 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -199,6 +199,14 @@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ +ifeq ($(TARGET),win32) +LIB_PREFIX= +LIB_SUFFIX=.lib +else +LIB_PREFIX=lib +LIB_SUFFIX=.a +endif + OMIT_OMIT_FP=no ifeq (@EMU_LOCK_CHECKING@,yes) @@ -279,15 +287,12 @@ endif ifeq ($(TARGET),win32) LIBS += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre -DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib else -LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a -DEPLIBS += \ - $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a -# rem liberts_internal.a +LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) endif +DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) + ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK PERFCTR_PATH=@PERFCTR_PATH@ @@ -305,7 +310,19 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS)) +ifneq ($(strip $(THR_LIB_NAME)),) +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ + $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) +else +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) +endif + +THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ + $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) + +LIBS += $(THR_LIBS) + +ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r) ifeq ($(findstring vxworks,$(TARGET)),vxworks) ERTS_INTERNAL_LIB=erts_internal @@ -317,7 +334,9 @@ ERTS_INTERNAL_LIB=erts_internal endif endif -LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) +LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) + +endif # erts_internal_r LIBS += @LIBRT@ diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index e2a79d6e4f..6b3c106a97 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock; #define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) #define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) #define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) -#define atom_init_lock() erts_smp_rwmtx_init(&atom_table_lock, \ - "atom_tab") + #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif @@ -304,12 +303,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_init(&atom_put_ops, 0); #endif - atom_init_lock(); + erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 28f69b9460..0815cdbc7f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -432,6 +432,7 @@ atom raw atom re atom re_pattern atom re_run_trap +atom read_concurrency atom ready_input atom ready_output atom ready_async diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 16ae643ed9..87503af7d5 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -64,6 +64,14 @@ #ifdef DEBUG static Uint install_debug_functions(void); +#if 0 +#define HARD_DEBUG +#ifdef __GNUC__ +#warning "* * * * * * * * * * * * * *" +#warning "* HARD DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * *" +#endif +#endif #endif extern void elib_ensure_initialized(void); @@ -391,6 +399,10 @@ refuse_af_strategy(struct au_init *init) static void init_thr_ix(int static_ixs); +#ifdef HARD_DEBUG +static void hdbg_init(void); +#endif + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -406,6 +418,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) ERTS_DEFAULT_ALCU_INIT }; +#ifdef HARD_DEBUG + hdbg_init(); +#endif + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); @@ -2862,12 +2878,10 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0a: - if (ethr_mutex_lock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: - if (ethr_mutex_unlock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); @@ -2883,16 +2897,13 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0e: - if (ethr_cond_broadcast((ethr_cond *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); - if (res != 0) - ERTS_ALC_TEST_ABORT; break; } case 0xf10: { @@ -2939,8 +2950,11 @@ unsigned long erts_alc_test(unsigned long op, #undef PRINT_OPS #endif - +#ifdef HARD_DEBUG +#define FENCE_SZ (4*sizeof(UWord)) +#else #define FENCE_SZ (3*sizeof(UWord)) +#endif #if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 @@ -2962,6 +2976,104 @@ unsigned long erts_alc_test(unsigned long op, #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) +#ifdef HARD_DEBUG + +#define ERL_ALC_HDBG_MAX_MBLK 100000 +#define ERTS_ALC_O_CHECK -1 + +typedef struct hdbg_mblk_ hdbg_mblk; +struct hdbg_mblk_ { + hdbg_mblk *next; + hdbg_mblk *prev; + void *p; + Uint s; + ErtsAlcType_t n; +}; + +static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; + +static hdbg_mblk *free_hdbg_mblks; +static hdbg_mblk *used_hdbg_mblks; +static erts_mtx_t hdbg_mblk_mtx; + +static void +hdbg_init(void) +{ + int i; + for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) + hdbg_mblks[i].next = &hdbg_mblks[i+1]; + hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; + free_hdbg_mblks = &hdbg_mblks[0]; + used_hdbg_mblks = NULL; + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); +} + +static void *check_memory_fence(void *ptr, + Uint *size, + ErtsAlcType_t n, + int func); +void erts_hdbg_chk_blks(void); + +void +erts_hdbg_chk_blks(void) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { + Uint sz; + check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); + ASSERT(sz == mblk->s); + } + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +static hdbg_mblk * +hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + mblk = free_hdbg_mblks; + if (!mblk) { + erts_fprintf(stderr, + "Ran out of debug blocks; please increase " + "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", + ERL_ALC_HDBG_MAX_MBLK); + abort(); + } + free_hdbg_mblks = mblk->next; + + mblk->p = p; + mblk->s = s; + mblk->n = n; + + mblk->next = used_hdbg_mblks; + mblk->prev = NULL; + if (used_hdbg_mblks) + used_hdbg_mblks->prev = mblk; + used_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); + return (void *) mblk; +} + +static void +hdbg_free(hdbg_mblk *mblk) +{ + erts_mtx_lock(&hdbg_mblk_mtx); + if (mblk->next) + mblk->next->prev = mblk->prev; + if (mblk->prev) + mblk->prev->next = mblk->next; + else + used_hdbg_mblks = mblk->next; + + mblk->next = free_hdbg_mblks; + free_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); @@ -3007,17 +3119,28 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { UWord *ui_ptr; UWord pattern; +#ifdef HARD_DEBUG + hdbg_mblk **mblkpp; +#endif if (!ptr) return NULL; ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); - + +#ifdef HARD_DEBUG + mblkpp = (hdbg_mblk **) ui_ptr++; +#endif + *(ui_ptr++) = sz; *(ui_ptr++) = pattern; memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); +#ifdef HARD_DEBUG + *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); +#endif + return (void *) ui_ptr; } @@ -3029,6 +3152,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) UWord pre_pattern; UWord post_pattern; UWord *ui_ptr; +#ifdef HARD_DEBUG + hdbg_mblk *mblk; +#endif if (!ptr) return NULL; @@ -3036,6 +3162,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); +#ifdef HARD_DEBUG + mblk = (hdbg_mblk *) *(--ui_ptr); +#endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { @@ -3091,6 +3220,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } +#ifdef HARD_DEBUG + switch (func) { + case ERTS_ALC_O_REALLOC: + case ERTS_ALC_O_FREE: + hdbg_free(mblk); + break; + default: + break; + } +#endif + return (void *) ui_ptr; } @@ -3103,6 +3243,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint dsize; void *res; +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); @@ -3132,13 +3276,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); - + res = set_memory_fence(res, size, n); #ifdef PRINT_OPS @@ -3168,6 +3316,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + } static Uint diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 6f88bbe5b8..7df9f19af0 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -212,7 +212,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED PROCESSES scheduler_data +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle @@ -246,6 +247,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib +type RDR_GRPS_MAP LONG_LIVED SYSTEM reader_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -269,7 +271,9 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +if threads -type ETHR_INTERNAL SYSTEM SYSTEM ethread_internal +type ETHR_STD STANDARD SYSTEM ethread_standard +type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived +type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived +ifnot smp diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 48cda52612..f582f4470f 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2563,6 +2563,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_sched_stat_term(BIF_P, 1)); } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) { BIF_RET(erts_nif_taints(BIF_P)); + } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) { + BIF_RET(erts_get_reader_groups_map(BIF_P)); } BIF_ERROR(BIF_P, BADARG); @@ -3426,6 +3428,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) { return erts_fake_scheduler_bindings(BIF_P, tp[2]); } + else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) { + Sint groups; + if (is_not_small(tp[2])) + BIF_ERROR(BIF_P, BADARG); + groups = signed_val(tp[2]); + if (groups < (Sint) 1 || groups > (Sint) INT_MAX) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); + } break; } default: @@ -3730,8 +3742,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - ethr_atomic_read(&stats->tries, (long *)&tries); - ethr_atomic_read(&stats->colls, (long *)&colls); + tries = (unsigned long) ethr_atomic_read(&stats->tries); + colls = (unsigned long) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index cbdaa459de..b0369a402b 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -78,11 +78,19 @@ enum DbIterSafety { ** The main meta table, containing all ets tables. */ #ifdef ERTS_SMP -# define META_MAIN_TAB_LOCK_CNT 16 -static union { - erts_smp_spinlock_t lck; - byte _cache_line_alignment[64]; -}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT]; + +#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 +#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) +#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) + +typedef union { + erts_smp_rwmtx_t rwmtx; + byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_smp_rwmtx_t))]; +} erts_meta_main_tab_lock_t; + +static erts_meta_main_tab_lock_t *meta_main_tab_locks; + #endif static struct { union { @@ -104,17 +112,13 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ -static ERTS_INLINE void meta_main_tab_lock(unsigned slot) +static ERTS_INLINE erts_smp_rwmtx_t * +get_meta_main_tab_lock(unsigned slot) { #ifdef ERTS_SMP - erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); -#endif -} - -static ERTS_INLINE void meta_main_tab_unlock(unsigned slot) -{ -#ifdef ERTS_SMP - erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); + return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; +#else + return NULL; #endif } @@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + LCK_NONE=4 } db_lock_kind_t; extern DbTableMethod db_hash; @@ -214,17 +219,17 @@ Export ets_select_continue_exp; */ static Export ets_delete_continue_exp; -static ERTS_INLINE DbTable* db_ref(DbTable* tb) +static ERTS_INLINE DbTable* db_ref(DbTable* tb, db_lock_kind_t kind) { - if (tb != NULL) { + if (tb != NULL && kind != LCK_READ) { erts_refc_inc(&tb->common.ref, 2); } return tb; } -static ERTS_INLINE DbTable* db_unref(DbTable* tb) +static ERTS_INLINE DbTable* db_unref(DbTable* tb, db_lock_kind_t kind) { - if (!erts_refc_dectest(&tb->common.ref, 0)) { + if (kind != LCK_READ && !erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", @@ -256,12 +261,19 @@ static ERTS_INLINE DbTable* db_unref(DbTable* tb) return tb; } -static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname) +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, + char *rwname, char* fixname) { +#ifdef ERTS_SMP + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + if (use_frequent_read_lock) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; +#endif erts_refc_init(&tb->common.ref, 1); erts_refc_init(&tb->common.fixref, 0); #ifdef ERTS_SMP - erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name); + erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, + rwname, tb->common.the_name); erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); #endif @@ -297,7 +309,7 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { - (void) db_ref(tb); + (void) db_ref(tb, kind); #ifdef ERTS_SMP db_lock_take_over_ref(tb, kind); #endif @@ -331,7 +343,7 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb); /* May delete table... */ + (void) db_unref(tb, kind); /* May delete table... */ } @@ -349,32 +361,49 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) } static ERTS_INLINE -DbTable* db_get_table(Process *p, - Eterm id, - int what, - db_lock_kind_t kind) +DbTable* db_get_table_aux(Process *p, + Eterm id, + int what, + db_lock_kind_t kind, + int meta_already_locked) { DbTable *tb = NULL; + erts_smp_rwmtx_t *mtl = NULL; if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - meta_main_tab_lock(slot); + if (!meta_already_locked) { + mtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rlock(mtl); + } +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + else { + erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) + || erts_lc_rwmtx_is_rwlocked(test_mtl)); + } +#endif if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { /* SMP: inc to prevent race, between unlock of meta_main_tab_lock * and the table locking outside the meta_main_tab_lock */ - tb = db_ref(meta_main_tab[slot].u.tb); + tb = db_ref(meta_main_tab[slot].u.tb, kind); } - meta_main_tab_unlock(slot); } else if (is_atom(id)) { - erts_smp_rwmtx_t* rwlock; - struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock); - erts_smp_rwmtx_rlock(rwlock); + struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); + if (!meta_already_locked) + erts_smp_rwmtx_rlock(mtl); + else{ + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); + mtl = NULL; + } + if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb); + tb = db_ref(bucket->pu.tb, kind); } } else { /* multi */ @@ -382,23 +411,33 @@ DbTable* db_get_table(Process *p, Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb); + tb = db_ref(bucket->pu.mvec[i].pu.tb, kind); break; } } } } - erts_smp_rwmtx_runlock(rwlock); } if (tb) { db_lock_take_over_ref(tb, kind); - if (tb->common.id == id && ((tb->common.status & what) != 0 || - p->id == tb->common.owner)) { - return tb; + if (tb->common.id != id + || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + db_unlock(tb, kind); + tb = NULL; } - db_unlock(tb, kind); } - return NULL; + if (mtl) + erts_smp_rwmtx_runlock(mtl); + return tb; +} + +static ERTS_INLINE +DbTable* db_get_table(Process *p, + Eterm id, + int what, + db_lock_kind_t kind) +{ + return db_get_table_aux(p, id, what, kind, 0); } /* Requires meta_main_tab_locks[slot] locked. @@ -413,15 +452,15 @@ static ERTS_INLINE void free_slot(int slot) erts_smp_spin_unlock(&meta_main_tab_main_lock); } -static int insert_named_tab(Eterm name_atom, DbTable* tb) +static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - - erts_smp_rwmtx_rwlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -468,17 +507,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } -static int remove_named_tab(Eterm name_atom) +static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; + Eterm name_atom = tb->common.id; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - erts_smp_rwmtx_rwlock(rwlock); +#ifdef ERTS_SMP + if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(rwlock); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + if (bucket->pu.tb == NULL) { goto done; } @@ -529,7 +583,8 @@ static int remove_named_tab(Eterm name_atom) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } @@ -1133,6 +1188,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1141,34 +1197,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { + + if (is_not_atom(BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } - if (is_not_atom(BIF_ARG_2)) { - goto badarg; + (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); + + if (is_small(BIF_ARG_1)) { + Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); + } + else if (is_atom(BIF_ARG_1)) { + (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (lck1 == lck2) + lck2 = NULL; + else if (lck1 > lck2) { + erts_smp_rwmtx_t *tmp = lck1; + lck1 = lck2; + lck2 = tmp; + } + } + else { + BIF_ERROR(BIF_P, BADARG); } + erts_smp_rwmtx_rwlock(lck1); + if (lck2) + erts_smp_rwmtx_rwlock(lck2); + + tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); + if (!tb) + goto badarg; + if (is_not_atom(tb->common.id)) { /* Not a named table */ tb->common.the_name = BIF_ARG_2; goto done; } - if (!insert_named_tab(BIF_ARG_2,tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) goto badarg; - } - if (!remove_named_tab(tb->common.id)) { + + if (!remove_named_tab(tb, 1)) erl_exit(1,"Could not find named tab %s", tb->common.id); - } tb->common.id = tb->common.the_name = BIF_ARG_2; done: ret = tb->common.id; db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: - db_unlock(tb, LCK_WRITE); + if (tb) + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1189,10 +1276,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked; + int is_named, is_fine_locked, frequent_read; int cret; DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; + erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1205,6 +1293,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = 1; is_named = 0; is_fine_locked = 0; + frequent_read = 0; heir = am_none; heir_data = (UWord) am_undefined; @@ -1238,6 +1327,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_fine_locked = 0; } else break; } + else if (tp[1] == am_read_concurrency) { + if (tp[2] == am_true) { + frequent_read = 1; + } else if (tp[2] == am_false) { + frequent_read = 0; + } else break; + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; @@ -1286,6 +1382,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +#ifdef ERTS_SMP + if (frequent_read && !(status & DB_PRIVATE)) + status |= DB_FREQ_READ; +#endif + /* we create table outside any table lock * and take the unusal cost of destroy table if it * fails to find a slot @@ -1308,7 +1409,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - db_init_lock(tb, "db_tab", "db_tab_fix"); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), + "db_tab", "db_tab_fix"); tb->common.keypos = keypos; tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); @@ -1351,15 +1453,17 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ - meta_main_tab_lock(slot); + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; ASSERT(IS_SLOT_ALIVE(slot)); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb)) { - meta_main_tab_lock(slot); + if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); free_slot(slot); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); db_lock_take_over_ref(tb,LCK_WRITE); free_heir_data(tb); @@ -1499,6 +1603,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1520,13 +1625,23 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif /* We must keep the slot, to be found by db_proc_dead() if process dies */ MARK_SLOT_DEAD(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); - if (is_atom(tb->common.id)) { - remove_named_tab(tb->common.id); - } + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); if (tb->common.owner != BIF_P->id) { DeclareTmpHeap(meta_tuple,3,BIF_P); @@ -1919,14 +2034,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) previous = NIL; j = 0; for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { - meta_main_tab_lock(i); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { j++; tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; } - meta_main_tab_unlock(i); + erts_smp_rwmtx_runlock(mmtl); } HRelease(BIF_P, hendp, hp); BIF_RET(previous); @@ -2630,12 +2746,30 @@ void init_db(void) size_t size; #ifdef ERTS_SMP - for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) { - erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i)); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, + (sizeof(erts_meta_main_tab_lock_t) + * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); + + if ((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) + meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) + ((((Uint) meta_main_tab_locks) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((Uint) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + + for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { + erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, + "meta_main_tab_slot", make_small(i)); } erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i)); + erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i)); } #endif @@ -2895,12 +3029,12 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb); /* while unlocked */ + db_ref(tb, LCK_NONE); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb); + tb = db_unref(tb, LCK_NONE); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -3008,12 +3142,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix)); + tb = db_ref(GET_ANY_SLOT_TAB(ix), LCK_WRITE); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; db_lock_take_over_ref(tb, LCK_WRITE); @@ -3045,7 +3180,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status |= DB_DELETE; if (is_atom(tb->common.id)) - remove_named_tab(tb->common.id); + remove_named_tab(tb, 0); free_heir_data(tb); free_fixations_locked(tb); @@ -3095,12 +3230,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb); + tb = db_ref(meta_main_tab[ix].u.tb, LCK_WRITE_REC); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int reds; DbFixation** pp; @@ -3274,6 +3410,7 @@ unlocked: #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ + db_ref(tb, LCK_WRITE); /* LCK_WRITE need it, but not LCK_READ */ erts_smp_rwmtx_runlock(&tb->common.rwlock); erts_smp_rwmtx_rwlock(&tb->common.rwlock); *kind_p = LCK_WRITE; @@ -3386,6 +3523,7 @@ static int free_table_cont(Process *p, int clean_meta_tab) { Eterm result; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG if (!first) { @@ -3411,9 +3549,20 @@ static int free_table_cont(Process *p, tb->common.id); #endif /* Completely done - we will not get called again. */ - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif free_slot(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); if (clean_meta_tab) { db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); @@ -3421,7 +3570,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb); + db_unref(tb, LCK_NONE); BUMP_REDS(p, 100); return 0; } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 124129a371..95ffc3afcd 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -623,12 +623,16 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; int i; + if (tb->common.type & DB_FREQ_READ) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i)); + erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", make_small(i)); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 382e5dceb5..672c5f2cd1 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -240,8 +240,9 @@ typedef struct db_table_common { #define DB_DUPLICATE_BAG (1 << 8) #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ +#define DB_FREQ_READ (1 << 11) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index aa37edafd1..d42820ddf3 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -186,10 +186,9 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_mutex_trylock()"); - return res; + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_trylock()"); + return ethr_mutex_trylock(&dmtx->mtx); #else return 0; #endif @@ -199,9 +198,9 @@ void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_lock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_lock()"); + ethr_mutex_lock(&dmtx->mtx); #endif } @@ -209,9 +208,9 @@ void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_unlock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_unlock()"); + ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -256,9 +255,9 @@ void erl_drv_cond_signal(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_signal()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_signal()"); + ethr_cond_signal(&dcnd->cnd); #endif } @@ -266,9 +265,9 @@ void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_broadcast()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_broadcast()"); + ethr_cond_broadcast(&dcnd->cnd); #endif } @@ -277,18 +276,13 @@ void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res; if (!dcnd || !dmtx) { - res = EINVAL; - error: - fatal_error(res, "erl_drv_cond_wait()"); + fatal_error(EINVAL, "erl_drv_cond_wait()"); } while (1) { - res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); + int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); if (res == 0) break; - if (res != EINTR) - goto error; } #endif } @@ -333,10 +327,9 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); + return ethr_rwmutex_tryrlock(&drwlck->rwmtx); #else return 0; #endif @@ -346,9 +339,9 @@ void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); + ethr_rwmutex_rlock(&drwlck->rwmtx); #endif } @@ -356,9 +349,9 @@ void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_runlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); + ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -366,10 +359,9 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrwlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); + return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #else return 0; #endif @@ -379,9 +371,9 @@ void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); + ethr_rwmutex_rwlock(&drwlck->rwmtx); #endif } @@ -389,9 +381,9 @@ void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwunlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); + ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 15d9538301..74e3ca048e 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock; #define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) #define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) #define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) -#define erts_fun_init_lock() erts_smp_rwmtx_init(&erts_fun_table_lock, \ - "fun_tab") static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -57,8 +55,12 @@ void erts_init_fun_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); - erts_fun_init_lock(); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 4a4507b212..14bd10b42c 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -78,6 +78,7 @@ static erts_tid_t main_thread; erts_cpu_info_t *erts_cpuinfo; +int erts_reader_groups; int erts_use_sender_punish; /* @@ -110,6 +111,7 @@ int erts_compat_rel; static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; +static int max_reader_groups; #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -505,6 +507,7 @@ void erts_usage(void) ERTS_MIN_COMPAT_REL, this_rel_num()); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); @@ -538,6 +541,50 @@ void erts_usage(void) erl_exit(-1, ""); } +#ifdef USE_THREADS +/* + * allocators for thread lib + */ + +static void *ethr_std_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size); +} +static void *ethr_std_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size); +} +static void ethr_std_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_STD, ptr); +} +static void *ethr_sl_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size); +} +static void *ethr_sl_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size); +} +static void ethr_sl_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_SL, ptr); +} +static void *ethr_ll_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size); +} +static void *ethr_ll_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size); +} +static void ethr_ll_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_LL, ptr); +} + +#endif + static void early_init(int *argc, char **argv) /* * Only put things here which are @@ -615,9 +662,15 @@ early_init(int *argc, char **argv) /* ? ncpuavail : (ncpuonln > 0 ? ncpuonln : no_schedulers)); +#ifdef ERTS_SMP + erts_max_main_threads = no_schedulers_online; +#endif + schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; + max_reader_groups = ERTS_MAX_READER_GROUPS; + if (argc && argv) { int i = 1; while (i < *argc) { @@ -627,6 +680,24 @@ early_init(int *argc, char **argv) /* } if (argv[i][0] == '-') { switch (argv[i][1]) { + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + char *arg = get_arg(sub_param+1, argv[i+1], &i); + if (sscanf(arg, "%d", &max_reader_groups) != 1) { + erts_fprintf(stderr, + "bad reader groups limit: %s\n", arg); + erts_usage(); + } + if (max_reader_groups < 0) { + erts_fprintf(stderr, + "bad reader groups limit: %d\n", + max_reader_groups); + erts_usage(); + } + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -699,6 +770,36 @@ early_init(int *argc, char **argv) /* erts_early_init_scheduling(); /* Require allocators */ erts_init_utils(); /* Require allocators */ +#ifdef USE_THREADS + { + erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; + elid.mem.std.alloc = ethr_std_alloc; + elid.mem.std.realloc = ethr_std_realloc; + elid.mem.std.free = ethr_std_free; + elid.mem.sl.alloc = ethr_sl_alloc; + elid.mem.sl.realloc = ethr_sl_realloc; + elid.mem.sl.free = ethr_sl_free; + elid.mem.ll.alloc = ethr_ll_alloc; + elid.mem.ll.realloc = ethr_ll_realloc; + elid.mem.ll.free = ethr_ll_free; + +#ifdef ERTS_SMP + elid.main_threads = erts_max_main_threads; +#else + elid.main_threads = 1; +#endif + elid.reader_groups = (elid.main_threads > 1 + ? elid.main_threads + : 0); + if (max_reader_groups <= 1) + elid.reader_groups = 0; + if (elid.reader_groups > max_reader_groups) + elid.reader_groups = max_reader_groups; + erts_reader_groups = elid.reader_groups; + + erts_thr_late_init(&elid); + } +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif @@ -1193,9 +1294,17 @@ erl_start(int argc, char **argv) erts_async_thread_suggested_stack_size)); break; - case 'r': - erts_ets_realloc_always_moves = 1; + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + get_arg(sub_param+1, argv[i+1], &i); + /* already handled */ + } + else { + erts_ets_realloc_always_moves = 1; + } break; + } case 'n': /* XXX obsolete */ break; case 'c': @@ -1280,6 +1389,7 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else + erts_thr_set_main_status(1, 1); set_main_stack_size(); process_main(); #endif @@ -1353,7 +1463,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) && !defined(ERTS_SMP) +#if defined(USE_THREADS) exit_async(); #endif #if HAVE_ERTS_MSEG diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index cee470ae37..99cc80e259 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,10 +96,10 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, - { "db_tab", "address" }, - { "db_tab_fix", "address" }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, @@ -119,9 +119,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "child_status", NULL }, #endif #ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, + { "sys_driver_data_lock", NULL }, #endif - { "drv_ev_state_grow", NULL, }, + { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, @@ -153,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "alcu_delayed_free", "index" }, { "mseg", NULL }, #ifdef HALFWORD_HEAP { "pmmap", NULL }, @@ -177,17 +178,19 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "pix_lock", "address" }, { "run_queues_lists", NULL }, { "sched_stat", NULL }, + { "run_queue_sleep_list", "address" }, #endif { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP - { "proc_lck_wtr_alloc", NULL }, + { "proc_lck_qs_alloc", NULL }, #endif #ifdef __WIN32__ #ifdef DEBUG { "save_ops_lock", NULL }, #endif #endif - { "mtrace_buf", NULL } + { "mtrace_buf", NULL }, + { "erts_alloc_hard_debug", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -199,6 +202,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { & ERTS_LC_FLG_LT_ALL \ & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +static __decl_noreturn void __noreturn lc_abort(void); + static char * lock_type(Uint16 flags) { @@ -222,7 +227,7 @@ rw_op_str(Uint16 flags) return " (r)"; case ERTS_LC_FLG_LO_WRITE: erts_fprintf(stderr, "\nInternal error\n"); - abort(); + lc_abort(); default: break; } @@ -271,28 +276,18 @@ static erts_lc_free_block_t *free_blocks; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS static ethr_spinlock_t free_blocks_lock; -#define ERTS_LC_LOCK ethr_spin_lock -#define ERTS_LC_UNLOCK ethr_spin_unlock -#else -static ethr_mutex free_blocks_lock; -#define ERTS_LC_LOCK ethr_mutex_lock -#define ERTS_LC_UNLOCK ethr_mutex_unlock -#endif static ERTS_INLINE void lc_lock(void) { - if (ERTS_LC_LOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_lock(&free_blocks_lock); } static ERTS_INLINE void lc_unlock(void) { - if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_unlock(&free_blocks_lock); } static ERTS_INLINE void lc_free(void *p) @@ -313,7 +308,7 @@ static void *lc_core_alloc(void) { lc_unlock(); erts_fprintf(stderr, "Lock checker out of memory!\n"); - abort(); + lc_abort(); } #else @@ -327,7 +322,7 @@ static void *lc_core_alloc(void) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - abort(); + lc_abort(); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -367,11 +362,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - abort(); + lc_abort(); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; @@ -513,7 +508,7 @@ uninitialized_lock(void) { erts_fprintf(stderr, "Performing operations on uninitialized lock!\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } static void @@ -523,7 +518,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -533,7 +528,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -541,7 +536,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -550,7 +545,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) print_lock("Lock order violation occured when locking ", lck, "!\n"); print_curr_locks(l_lcks); print_lock_order(); - abort(); + lc_abort(); } static void @@ -561,7 +556,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, print_lock(op, lck, "!\n"); ASSERT(l_lcks); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -613,7 +608,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, } } print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -621,7 +616,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -629,7 +624,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l { print_lock("Unrequire on ", lck, " lock not required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -637,7 +632,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -645,7 +640,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } @@ -658,13 +653,23 @@ thread_exit_handler(void) erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } destroy_locked_locks(l_lcks); /* erts_tsd_set(locks_key, NULL); */ } } +static __decl_noreturn void +lc_abort(void) +{ +#ifdef __WIN32__ + DebugBreak(); +#else + abort(); +#endif +} + void erts_lc_set_thread_name(char *thread_name) { @@ -676,7 +681,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); } } @@ -686,7 +691,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion) erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n", file, line, assertion); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); return 0; } @@ -699,7 +704,7 @@ void erts_lc_fail(char *fmt, ...) va_end(args); erts_fprintf(stderr, "\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } @@ -719,7 +724,7 @@ erts_lc_get_lock_order_id(char *name) "(update erl_lock_check.c)\n", name); } - abort(); + lc_abort(); return (Sint16) -1; } @@ -895,6 +900,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } } +void +erts_lc_check_no_locked_of_type(Uint16 flags) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + if (l_lcks) { + erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { + if (l_lck->flags & flags) { + erts_fprintf(stderr, + "Locked lock of type %s found which isn't " + "allowed here!\n", + lock_type(l_lck->flags)); + print_curr_locks(l_lcks); + lc_abort(); + } + } + } +} + int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) { @@ -1283,13 +1307,8 @@ erts_lc_init(void) free_blocks = NULL; #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ -#ifdef ETHR_HAVE_NATIVE_LOCKS if (ethr_spinlock_init(&free_blocks_lock) != 0) - abort(); -#else - if (ethr_mutex_init(&free_blocks_lock) != 0) - abort(); -#endif + lc_abort(); erts_tsd_key_create(&locks_key); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index d5e2ede9ac..0372e6850d 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); +void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 26028aeefc..239773f366 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -166,8 +166,8 @@ static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char int i; type = lcnt_lock_type(lock->flag); - ethr_atomic_read(&lock->r_state, &r_state); - ethr_atomic_read(&lock->w_state, &w_state); + r_state = ethr_atomic_read(&lock->r_state); + w_state = ethr_atomic_read(&lock->w_state); if (lock->flag & flag) { @@ -394,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { ASSERT(eltd); - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); if (option & ERTS_LCNT_LO_WRITE) { - ethr_atomic_read(&lock->r_state, &r_state); + r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); } if (option & ERTS_LCNT_LO_READ) { @@ -423,7 +423,7 @@ void erts_lcnt_lock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc( &lock->w_state); eltd = lcnt_get_thread_data(); @@ -478,7 +478,7 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); } @@ -522,12 +522,12 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG /* flowstate */ - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 1); ethr_atomic_dec( &lock->flowstate); /* write state */ - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ASSERT(w_state > 0) #endif ethr_atomic_dec(&lock->w_state); @@ -558,7 +558,7 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { if (res != EBUSY) { #ifdef DEBUG - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); #endif diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 9cf55ee319..b1478758a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename) Uint16 port; erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_set_forksafe(&mtrace_buf_mutex); erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); - erts_mtx_set_forksafe(&mtrace_op_mutex); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 5865d33138..a3a1d95cd3 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) return dep_tmpl; @@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl) dep->prev = NULL; erts_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr); + erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; dep->connection_id = 0; @@ -580,6 +582,18 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) ErlNode ne; ne.sysname = sysname; ne.creation = creation; + + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + res = hash_get(&erts_node_table, (void *) &ne); + if (res && res != erts_this_node) { + long refc = erts_refc_inctest(&res->refc, 0); + if (refc < 2) /* New or pending delete */ + erts_refc_inc(&res->refc, 1); + } + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + if (res) + return res; + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); @@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; HashFunctions f; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; @@ -719,9 +737,10 @@ void erts_init_node_tables(void) erts_this_dist_entry->prev = NULL; erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, + &rwmtx_opt, + "dist_entry", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->sysname = am_Noname; erts_this_dist_entry->cid = NIL; erts_this_dist_entry->connection_id = 0; @@ -772,8 +791,8 @@ void erts_init_node_tables(void) (void) hash_put(&erts_node_table, (void *) erts_this_node); - erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table"); - erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table"); + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); references_atoms_need_init = 1; } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 967a14f0d1..c10724b951 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id, if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); + erts_smp_runq_unlock(runq); } else { enqueue_port(runq, pp); @@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id, profile_runnable_port(pp, am_active); } + erts_smp_runq_unlock(runq); + erts_smp_notify_inc_runq(runq); } - erts_smp_runq_unlock(runq); return 0; } @@ -944,8 +946,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; - erts_smp_notify_inc_runq(xrunq); erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; @@ -1112,7 +1114,6 @@ erts_port_migrate(Port *prt, int *prt_locked, dequeue_port(from_rq, prt); erts_smp_atomic_set(&prt->run_queue, (long) to_rq); enqueue_port(to_rq, prt); - erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 41031f5468..abf4c9d96a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -46,7 +46,13 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 +#define ERTS_SCHED_SPIN_UNTIL_YIELD 100 + +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ + (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) +#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 #define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) #define ERTS_WAKEUP_OTHER_DEC 10 @@ -106,6 +112,10 @@ Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; +#ifdef ERTS_SMP +Uint erts_max_main_threads; +#endif + int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -116,16 +126,34 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHED_CHANGING_ONLINE 1 -#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((long) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((long) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((long) 1) << 2) + +#ifndef DEBUG + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic_set(&schdlr_sspnd.changing, (VAL)) + +#else + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ +do { \ + long old_val__ = erts_smp_atomic_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) + +#endif + static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int changing; int online; int curr_online; int wait_curr_online; + erts_smp_atomic_t changing; erts_smp_atomic_t active; struct { erts_smp_atomic_t ongoing; @@ -231,6 +259,17 @@ typedef union { ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +#ifdef ERTS_SMP + +typedef union { + ErtsSchedulerSleepInfo ssi; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; +} ErtsAlignedSchedulerSleepInfo; + +static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; + +#endif + #ifndef BM_COUNTERS static int processes_busy; #endif @@ -288,8 +327,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -353,6 +399,11 @@ static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size #endif +static int reader_group_lookup(int logical); +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + static void early_cpu_bind_init(void); static void late_cpu_bind_init(void); @@ -582,6 +633,76 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +void +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, long flags) +{ + switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_sys_schedule_interrupt(1); + break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_tse_set(ssi->event); + break; + case 0: + break; + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + __FILE__, __LINE__); + break; + } +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + + for (i = 0; i < erts_no_schedulers; i++) { + long aux_work; + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(i); + aux_work = erts_smp_atomic_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) + erts_sched_poke(ssi); + } +} +#endif + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +static ERTS_INLINE long +blockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = erts_smp_atomic_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; + erts_check_children(); + } +#endif + } + return aux_work; +} + +#endif + +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +static ERTS_INLINE long +nonblockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + long aux_work) +{ + if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + + } +} +#endif + static void prepare_for_block(void *vrq) { @@ -634,7 +755,31 @@ erts_active_schedulers(void) return as; } +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ #ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + erts_smp_atomic_set(&doing_sys_schedule, 0); + } + return 0; +#else + return !erts_port_task_have_outstanding_io_tasks(); +#endif +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; +} static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) @@ -695,101 +840,304 @@ non_empty_runq(ErtsRunQueue *rq) } } -static ERTS_INLINE int -sched_spin_wake(ErtsRunQueue *rq) +static long +sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = 0; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_atomic_inc(&rq->spin_wake); - return 1; - } - return 0; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED)); + return oflgs; } -static ERTS_INLINE int -sched_spin_wake_all(ErtsRunQueue *rq) +static long +sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + long xflgs = ERTS_SSI_FLG_WAITING; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) - erts_smp_atomic_add(&rq->spin_wake, val); - return val; -#endif + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } while (oflgs & ERTS_SSI_FLG_WAITING); + return oflgs; +} + +static long +sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + long until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; } +static long +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, long sleep_type) +{ + long oflgs; + long nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + + if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + return oflgs; + } + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } +} + +#define ERTS_SCHED_WAIT_WOKEN(FLGS) \ + (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ + != ERTS_SSI_FLG_WAITING) + static void -sched_sys_wait(Uint no, ErtsRunQueue *rq) +scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { - long dt; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + int spincount; + long flgs; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_smp_spin_lock(&rq->sleepers.lock); + flgs = sched_prep_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + /* Go suspend instead... */ + erts_smp_spin_unlock(&rq->sleepers.lock); + return; + } + + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + + /* + * If all schedulers are waiting, one of them *should* + * be waiting in erl_sys_schedule() + */ + + if (!prepare_for_sys_schedule()) { + + sched_waiting(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + tse_wait: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + tse_blockable_aux_work: + aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - sched_waiting_sys(no, rq); + while (1) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif - erl_sys_schedule(1); /* Might give us something to do */ + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + + flgs = sched_prep_cont_spin_wait(ssi); + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto tse_blockable_aux_work; + } +#endif - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_runq_unlock(rq); } - } - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + erts_smp_runq_lock(rq); + sched_active(esdp->no, rq); + } else { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + long dt; + + erts_smp_atomic_set(&function_calls, 0); + *fcalls = 0; + + sched_waiting_sys(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + + while (spincount-- > 0) { + + sys_poll_aux_work: + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = do_time_read_and_reset(); + if (dt) bump_timer(dt); + + sys_aux_work: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + } + + /* + * If we got new I/O tasks we aren't allowed to + * call erl_sys_schedule() until it is handled. + */ + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to continue checking for I/O... + */ + if (!prepare_for_sys_schedule()) { + spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + goto tse_wait; + } + } + } + + erts_smp_runq_lock(rq); + /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). */ - if (!erts_port_task_have_outstanding_io_tasks()) { -#endif + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to wait in erl_sys_schedule() after all... + */ + if (prepare_for_sys_schedule()) + goto do_sys_schedule; + + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + else { + do_sys_schedule: erts_sys_schedule_interrupt(0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } + + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_smp_runq_unlock(rq); erl_sys_schedule(0); @@ -797,134 +1145,103 @@ sched_sys_wait(Uint no, ErtsRunQueue *rq) dt = do_time_read_and_reset(); if (dt) bump_timer(dt); - erts_smp_runq_lock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + erts_smp_atomic_set(&doing_sys_schedule, 0); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + sched_active_sys(esdp->no, rq); } } -#endif - sched_active_sys(no, rq); -} - -static void -sched_cnd_wait(Uint no, ErtsRunQueue *rq) -{ -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif - - sched_waiting(no, rq); - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); -#else - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_mtx_unlock(&rq->mtx); - - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_mtx_unlock(&rq->mtx); - } - } - - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val == 0) { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); - } - else { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - } -#endif - - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - - sched_active(no, rq); } -static void -wake_one_scheduler(void) -{ - ASSERT(erts_common_run_queue); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); - if (erts_common_run_queue->waiting) { - if (!sched_spin_wake(erts_common_run_queue)) { - if (erts_common_run_queue->waiting == -1) /* One scheduler waiting - and doing so in - sys_schedule */ - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&erts_common_run_queue->cnd); - } +static ERTS_INLINE long +ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) +{ + /* reset all flags but suspended */ + long oflgs; + long nflgs = 0; + long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return oflgs; + nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + xflgs = oflgs; } } static void -wake_scheduler(ErtsRunQueue *rq, int incq) +wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - ASSERT(!erts_common_run_queue); - ASSERT(-1 <= rq->waiting && rq->waiting <= 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->waiting && !rq->woken) { - if (!sched_spin_wake(rq)) { - if (rq->waiting < 0) - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&rq->cnd); + int res; + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; + + /* + * The unlocked run queue is not strictly necessary + * from a thread safety or deadlock prevention + * perspective. It will, however, cost us performance + * if it is locked during wakup of another scheduler, + * so all code *should* handle this without having + * the lock on the run queue. + */ + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + sl = &rq->sleepers; + + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) + erts_smp_spin_unlock(&sl->lock); + else if (one) { + long flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; } - rq->woken = 1; - if (incq) + if (ssi->next) + ssi->next->prev = ssi->prev; + + res = sl->list != NULL; + erts_smp_spin_unlock(&sl->lock); + + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + + if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) non_empty_runq(rq); } + else { + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } static void wake_all_schedulers(void) { - if (erts_common_run_queue) { - erts_smp_runq_lock(erts_common_run_queue); - if (erts_common_run_queue->waiting) { - if (erts_common_run_queue->waiting < 0) - erts_sys_schedule_interrupt(1); - sched_spin_wake_all(erts_common_run_queue); - erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); - } - erts_smp_runq_unlock(erts_common_run_queue); - } + if (erts_common_run_queue) + wake_scheduler(erts_common_run_queue, 0, 0); else { int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - wake_scheduler(rq, 0); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); } } } @@ -939,14 +1256,14 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) wrq = ERTS_RUNQ_IX(ix); iflgs = erts_smp_atomic_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { - erts_smp_xrunq_lock(crq, wrq); if (activate) { if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0); - erts_smp_xrunq_unlock(crq, wrq); + wake_scheduler(wrq, 0, 1); return 1; } return 0; @@ -992,15 +1309,13 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (erts_common_run_queue) - wake_one_scheduler(); - else - wake_scheduler(runq, 1); + if (runq) + wake_scheduler(runq, 1, 1); #endif } void -erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +erts_smp_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } @@ -1146,20 +1461,23 @@ static void evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) { Port *prt; + int notify_to_rq = 0; int prio; int prt_locked = 0; int rq_locked = 0; int evac_rq_locked = 1; + ErtsMigrateResult mres; erts_smp_runq_lock(evac_rq); + erts_smp_atomic_bor(&evac_rq->scheduler->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1187,9 +1505,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) /* Evacuate scheduled ports */ prt = evac_rq->ports.start; while (prt) { - (void) erts_port_migrate(prt, &prt_locked, + mres = erts_port_migrate(prt, &prt_locked, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (prt_locked) erts_smp_port_unlock(prt); if (!evac_rq_locked) { @@ -1218,9 +1538,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) goto end_of_proc; } - (void) erts_proc_migrate(proc, &proc_locks, + mres = erts_proc_migrate(proc, &proc_locks, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); if (!evac_rq_locked) { @@ -1252,10 +1574,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (rq_locked) erts_smp_runq_unlock(rq); - if (!evac_rq_locked) - erts_smp_runq_lock(evac_rq); - wake_scheduler(evac_rq, 0); - erts_smp_runq_unlock(evac_rq); + if (evac_rq_locked) + erts_smp_runq_unlock(evac_rq); + + if (notify_to_rq) + smp_notify_inc_runq(rq); + + wake_scheduler(evac_rq, 0, 1); } static int @@ -1483,31 +1808,6 @@ try_steal_task(ErtsRunQueue *rq) return res; } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) { - erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); - ERTS_SCHEDULER_IX(i)->check_children = 1; - if (!erts_common_run_queue) - wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); - erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); - } - if (ongoing_multi_scheduling_block()) { - /* Also blocked schedulers need to check children */ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - for (i = 0; i < erts_no_schedulers; i++) - ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } - if (erts_common_run_queue) - wake_all_schedulers(); -} -#endif - /* Run queue balancing */ typedef struct { @@ -2100,6 +2400,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_atomic_init(&no_empty_run_queues, 0); #endif + erts_no_run_queues = n; + for (ix = 0; ix < n; ix++) { int pix, rix; ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); @@ -2114,8 +2416,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); - erts_smp_atomic_init(&rq->spin_waiter, 0); - erts_smp_atomic_init(&rq->spin_wake, 0); +#ifdef ERTS_SMP + erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif rq->waiting = 0; rq->woken = 0; @@ -2166,7 +2470,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - erts_no_run_queues = n; #ifdef ERTS_SMP @@ -2181,9 +2484,34 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #endif + n = (int) no_schedulers; + erts_no_schedulers = n; + +#ifdef ERTS_SMP + /* Create and initialize scheduler sleep info */ + + aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, + (sizeof(ErtsAlignedSchedulerSleepInfo) + *(n+1))); + if ((((Uint) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) + aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) + ((((Uint) aligned_sched_sleep_info) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + for (ix = 0; ix < n; ix++) { + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); +#if 0 /* no need to initialize these... */ + ssi->next = NULL; + ssi->prev = NULL; +#endif + erts_smp_atomic_init(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_thread_func */ + erts_smp_atomic_init(&ssi->aux_work, 0); + } +#endif + /* Create and initialize scheduler specific data */ - n = (int) no_schedulers; erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, (sizeof(ErtsAlignedSchedulerData) *(n+1))); @@ -2200,6 +2528,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; #if HALFWORD_HEAP /* Registers need to be heap allocated (correct memory range) for tracing to work */ @@ -2228,11 +2557,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->check_children = 0; - esdp->blocked_check_children = 0; -#endif - erts_smp_atomic_init(&esdp->suspended, 0); erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2241,7 +2565,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; + erts_smp_atomic_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); @@ -2264,7 +2588,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2275,7 +2600,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.wait_curr_online = no_schedulers_online; schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); erts_smp_atomic_init(&doing_sys_schedule, 0); @@ -2423,13 +2749,115 @@ susp_sched_resume_block(void *unused) } static void +scheduler_ix_resume_wake(Uint ix) +{ + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long oflgs; + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, 0, xflgs); + if (oflgs == xflgs) { + erts_sched_finish_poke(ssi, oflgs); + break; + } + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); +} + +static long +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, long xpct) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = xpct; + + do { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + + return oflgs; +} + +static long +sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + long flgs; + + do { + flgs = erts_smp_atomic_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static long +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +{ + long oflgs; + long nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + long xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + return oflgs; + } + xflgs = oflgs; + } +} + +static void suspend_scheduler(ErtsSchedulerData *esdp) { + long flgs; + int changing; long no = (long) esdp->no; ErtsRunQueue *rq = esdp->run_queue; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; + int reset_read_group = 0; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + long aux_work; +#endif /* * Schedulers may be suspended in two different ways: @@ -2451,113 +2879,151 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (scheduler2cpu_map[esdp->no].bound_id >= 0 && erts_unbind_from_cpu(erts_cpuinfo) == 0) { esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + reset_read_group = 1; } erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (reset_read_group) + erts_smp_rwmtx_set_reader_group(0); + + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(0, 0); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) - schdlr_sspnd.changing = 0; - } - - while (1) { + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; - erts_smp_runq_lock(esdp->run_queue); - check_children = esdp->check_children; - esdp->check_children = 0; - erts_smp_runq_unlock(esdp->run_queue); - if (check_children) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_check_children(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } } -#endif - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { - int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; - curr_online = 0; - changed = 1; + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + changing = erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; - curr_online = 1; - changed = 1; + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; } - if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) - schdlr_sspnd.changing = 0; - } - if (wake) { - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - wake = 0; - } + flgs = erts_smp_atomic_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + blockable_aux_work: + blockable_aux_work(esdp, ssi, aux_work); +#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (1) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + while (1) { + long flgs; +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->blocked_check_children) - break; + flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; + + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto blockable_aux_work; + } #endif - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + } + + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) - break; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + } - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; + active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && schdlr_sspnd.online == active_schedulers) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->blocked_check_children = 0; -#endif + ASSERT(no <= schdlr_sspnd.online); + ASSERT(!erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); } - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED - && schdlr_sspnd.online == active_schedulers) { - schdlr_sspnd.changing = 0; - } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(curr_online); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (esdp->no <= erts_max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -2622,8 +3088,10 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; + long changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (yield_allowed && schdlr_sspnd.changing) + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { *active = *online = schdlr_sspnd.online; @@ -2643,6 +3111,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; + long changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -2652,7 +3121,8 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } else { @@ -2661,17 +3131,19 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); schdlr_sspnd.online = no; if (no > online) { int ix; schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) - /* No schedulers to resume */; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } else if (erts_common_run_queue) { for (ix = online; ix < no; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 0); + scheduler_ix_resume_wake(ix); } else { if (plocks) { @@ -2685,6 +3157,7 @@ erts_set_schedulers_online(Process *p, erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* * Spread evacuation paths among all online @@ -2699,7 +3172,6 @@ erts_set_schedulers_online(Process *p, erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); res = ERTS_SCHDLR_SSPND_DONE; } else /* if (no < online) */ { @@ -2716,12 +3188,17 @@ erts_set_schedulers_online(Process *p, schdlr_sspnd.wait_curr_online = no+1; } - if (ongoing_multi_scheduling_block()) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - else if (erts_common_run_queue) { + if (ongoing_multi_scheduling_block()) { for (ix = no; ix < online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 1); + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) { + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } wake_all_schedulers(); } else { @@ -2748,7 +3225,10 @@ erts_set_schedulers_online(Process *p, erts_smp_atomic_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); + } } } @@ -2762,6 +3242,12 @@ erts_set_schedulers_online(Process *p, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -2776,11 +3262,12 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; + long changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } else if (on) { /* ------ BLOCK ------ */ @@ -2794,19 +3281,22 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { + int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + ASSERT(0 == erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); - if (schdlr_sspnd.online == 1) { + if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); if (p->scheduler_data->no == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 1; @@ -2820,17 +3310,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + for (ix = 1; ix < online; ix++) + erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); erts_smp_atomic_set(&balance_info.used_runqs, 1); - for (ix = 0; ix < schdlr_sspnd.online; ix++) { + for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); erts_smp_runq_unlock(rq); } @@ -2855,6 +3347,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic_read(&schdlr_sspnd.changing))); + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -2898,7 +3397,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.msb.procs) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); #ifdef DEBUG ERTS_FOREACH_RUNQ(rq, { @@ -2925,13 +3424,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); - schdlr_sspnd.changing = 0; + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + erts_smp_atomic_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2948,6 +3447,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* Spread evacuation paths among all online run queues */ @@ -2963,7 +3463,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -3035,18 +3534,34 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { +#ifdef ERTS_SMP + Uint no = ((ErtsSchedulerData *) vesdp)->no; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; - Uint no = ((ErtsSchedulerData *) vesdp)->no; erts_snprintf(&buf[0], 31, "scheduler %bpu", no); erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP + + if (no <= erts_max_main_threads) { + erts_thr_set_main_status(1, (int) no); + if (erts_reader_groups) { + int rg = (int) no; + if (rg > erts_reader_groups) + rg = (((int) no) - 1) % erts_reader_groups + 1; + erts_smp_rwmtx_set_reader_group(rg); + } + } + erts_proc_lock_prepare_proc_lock_waiter(); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + + #endif erts_register_blockable_thread(); #ifdef HIPE @@ -3055,30 +3570,30 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + ASSERT(erts_smp_atomic_read(&schdlr_sspnd.changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); - schdlr_sspnd.curr_online--; + if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { + erts_smp_atomic_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + } - if (((ErtsSchedulerData *) vesdp)->no != 1) { - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - schdlr_sspnd.changing = 0; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); } - } - else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - schdlr_sspnd.changing = 0; - else { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - ASSERT(!schdlr_sspnd.changing); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3428,6 +3943,7 @@ processor_order_cmp(const void *vx, const void *vy) static void check_cpu_bind(ErtsSchedulerData *esdp) { + int rg = 0; int res; int cpu_id; erts_smp_runq_unlock(esdp->run_queue); @@ -3459,6 +3975,12 @@ check_cpu_bind(ErtsSchedulerData *esdp) erts_send_error_to_logger_nogl(dsbufp); } } + if (erts_reader_groups) { + if (esdp->cpu_id >= 0) + rg = reader_group_lookup(esdp->cpu_id); + else + rg = (((int) esdp->no) - 1) % erts_reader_groups + 1; + } erts_smp_runq_lock(esdp->run_queue); #ifdef ERTS_SMP if (erts_common_run_queue) @@ -3469,6 +3991,8 @@ check_cpu_bind(ErtsSchedulerData *esdp) #endif erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + if (erts_reader_groups) + erts_smp_rwmtx_set_reader_group(rg); } static void @@ -3497,11 +4021,13 @@ signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) wake_all_schedulers(); } else { - ERTS_FOREACH_RUNQ(rq, - { + for (s_ix = 0; s_ix < erts_no_run_queues; s_ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(s_ix); + erts_smp_runq_lock(rq); rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - wake_scheduler(rq, 0); - }); + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; } #else check_cpu_bind(erts_get_scheduler_data()); @@ -3541,6 +4067,490 @@ erts_init_scheduler_bind_type(char *how) return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; } +/* + * reader groups map + */ + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int *map; + int size; + int groups; +} erts_reader_groups_map_test; + +typedef struct { + int id; + int sub_levels; + int reader_groups; +} erts_rg_count_t; + +typedef struct { + int logical; + int reader_group; +} erts_reader_groups_map_t; + +typedef struct { + erts_reader_groups_map_t *map; + int map_size; + int logical_processors; + int groups; +} erts_make_reader_groups_map_test; + +static int reader_groups_available_cpu_check; +static int reader_groups_logical_processors; +static int reader_groups_map_size; +static erts_reader_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_RG ERTS_TOPOLOGY_MAX_DEPTH + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test); + +static Eterm +get_reader_groups_map(Process *c_p, + erts_reader_groups_map_t *map, + int map_size, + int logical_processors) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + logical_processors*(2+3); +#endif + for (i = map_size - 1; i >= 0; i--) { + if (map[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map[i].logical), + make_small(map[i].reader_group)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_make_reader_groups_map_test test; + + test.groups = groups; + make_reader_groups_map(&test); + if (!test.map) + res = NIL; + else { + res = get_reader_groups_map(c_p, + test.map, + test.map_size, + test.logical_processors); + erts_free(ERTS_ALC_T_TMP, test.map); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + res = get_reader_groups_map(c_p, + reader_groups_map, + reader_groups_map_size, + reader_groups_logical_processors); + erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(erts_cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_RG] = 0; + + ASSERT(last.logical != cpudata[a].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static int +reader_group_lookup(int logical) +{ + int start = logical % reader_groups_map_size; + int ix = start; + + do { + if (reader_groups_map[ix].logical == logical) { + ASSERT(reader_groups_map[ix].reader_group > 0); + return reader_groups_map[ix].reader_group; + } + ix++; + if (ix == reader_groups_map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +reader_group_insert(erts_reader_groups_map_t *map, int map_size, + int logical, int reader_group) +{ + int start = logical % map_size; + int ix = start; + + do { + if (map[ix].logical < 0) { + map[ix].logical = logical; + map[ix].reader_group = reader_group; + return; + } + ix++; + if (ix == map_size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_rg_count_t *rgc, int level, int aix, int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + rgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + rgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && rgc->id == avail[aix].level[level]); + rgc->reader_groups = 0; + return aix; +} + +static int +write_reader_groups(int *rgp, erts_rg_count_t *rgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int rg = *rgp; + int sub_level = level+1; + int sl_per_gr = rgcp->sub_levels / rgcp->reader_groups; + int xsl = rgcp->sub_levels % rgcp->reader_groups; + int sls = 0; + int last = -1; + int xsl_rg_lim = (rgcp->reader_groups - xsl) + rg + 1; + + ASSERT(level < 0 || avail[a].level[level] == rgcp->id) + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + rg++; + if (rg >= xsl_rg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_RG] = rg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == rgcp->id)); + + ASSERT(rgcp->reader_groups == rg - *rgp); + + *rgp = rg; + + return a; +} + +static int +rg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +rg_count_id_compare(const void *vx, const void *vy) +{ + erts_rg_count_t *x = (erts_rg_count_t *) vx; + erts_rg_count_t *y = (erts_rg_count_t *) vy; + return x->id - y->id; +} + +static void +make_reader_groups_map(erts_make_reader_groups_map_test *test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + erts_reader_groups_map_t *map; + int map_sz; + int groups = erts_reader_groups; + + if (test) { + test->map = NULL; + test->map_size = 0; + groups = test->groups; + } + + if (!groups) + return; + + if (!test) { + if (reader_groups_map) + erts_free(ERTS_ALC_T_RDR_GRPS_MAP, reader_groups_map); + + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; + } + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test != NULL); + + destroy_tmp_cpu_topology_copy(cpudata); + + map_sz = avail_sz*2+1; + + if (test) { + map = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + test->map = map; + test->map_size = map_sz; + test->logical_processors = avail_sz; + } + else { + map = erts_alloc(ERTS_ALC_T_RDR_GRPS_MAP, + (sizeof(erts_reader_groups_map_t) + * map_sz)); + reader_groups_map = map; + reader_groups_logical_processors = avail_sz; + reader_groups_map_size = map_sz; + + } + + for (i = 0; i < map_sz; i++) { + map[i].logical = -1; + map[i].reader_group = 0; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= groups) { + int a, rg, last = -1; + rg = 0; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + rg++; + last = avail[a].level[spread_level]; + } + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + rg); + } + } + else { /* groups < no.level[spread_level] */ + erts_rg_count_t *rg_count; + int a, rg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + rg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_rg_count_t)); + + if (toplevels == 1) { + rg_count[0].id = 0; + rg_count[0].sub_levels = no.level[spread_level]; + rg_count[0].reader_groups = groups; + } + else { + int rgs_per_tl, rgs; + rgs = groups; + rgs_per_tl = rgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + rg_count[i].id = avail[a].level[tl]; + a = sub_levels(&rg_count[i], tl, a, avail_sz, avail); + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels < rgs_per_tl) { + rg_count[i].reader_groups = rg_count[i].sub_levels; + rgs -= rg_count[i].sub_levels; + } + else { + rg_count[i].reader_groups = rgs_per_tl; + rgs -= rgs_per_tl; + } + } + + while (rgs > 0) { + for (i = 0; i < toplevels; i++) { + if (rg_count[i].sub_levels == rg_count[i].reader_groups) + break; + else { + rg_count[i].reader_groups++; + if (--rgs == 0) + break; + } + } + } + + qsort(rg_count, + toplevels, + sizeof(erts_rg_count_t), + rg_count_id_compare); + } + + a = i = rg = 0; + while (a < avail_sz) { + a = write_reader_groups(&rg, &rg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(groups == rg); + + for (a = 0; a < avail_sz; a++) + reader_group_insert(map, + map_sz, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_RG]); + + erts_free(ERTS_ALC_T_TMP, rg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +/* + * CPU topology + */ + typedef struct { int *id; int used; @@ -4054,6 +5064,8 @@ erts_set_cpu_topology(Process *c_p, Eterm term) sizeof(erts_cpu_topology_t)*cpudata_size); } + make_reader_groups_map(NULL); + signal_schedulers_bind_change(cpudata, cpudata_size); done: @@ -4457,6 +5469,11 @@ early_cpu_bind_init(void) cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + reader_groups_available_cpu_check = 1; + reader_groups_logical_processors = 0; + reader_groups_map_size = 0; + reader_groups_map = NULL; + if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, system_cpudata_size)) { @@ -4492,6 +5509,8 @@ late_cpu_bind_init(void) : ERTS_CPU_BIND_NONE); } + make_reader_groups_map(NULL); + if (cpu_bind_order != ERTS_CPU_BIND_NONE) { erts_cpu_topology_t *cpudata; int cpudata_size; @@ -5357,7 +6376,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p) } /* schedule a process */ -static ERTS_INLINE void +static ERTS_INLINE ErtsRunQueue * internal_add_to_runq(ErtsRunQueue *runq, Process *p) { Uint32 prev_status = p->status; @@ -5368,12 +6387,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return; + return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { ASSERT(p->status != P_SUSPENDED); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return; + return NULL; } ASSERT(!p->scheduler_data); #endif @@ -5412,20 +6431,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) profile_runnable_proc(p, am_active); } - smp_notify_inc_runq(add_runq); - if (add_runq != runq) erts_smp_runq_unlock(add_runq); + + return add_runq; } void erts_add_to_runq(Process *p) { + ErtsRunQueue *notify_runq; ErtsRunQueue *runq = erts_get_runq_proc(p); erts_smp_runq_lock(runq); - internal_add_to_runq(runq, p); + notify_runq = internal_add_to_runq(runq, p); erts_smp_runq_unlock(runq); + smp_notify_inc_runq(notify_runq); + } /* Possibly remove a scheduled process we need to suspend */ @@ -5564,8 +6586,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); - smp_notify_inc_runq(to_rq); - return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -5762,30 +6782,6 @@ erts_set_process_priority(Process *p, Eterm new_value) return old_value; } -#ifdef ERTS_SMP - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - return 0; -} - -#else - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - return !erts_port_task_have_outstanding_io_tasks(); -} - -#endif - /* note that P_RUNNING is only set so that we don't try to remove ** running processes from the schedule queue if they exit - a running ** process not being in the schedule queue!! @@ -5920,8 +6916,11 @@ Process *schedule(Process *p, int calls) p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + ErtsRunQueue *notify_runq; p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); + if (notify_runq != rq) + smp_notify_inc_runq(notify_runq); } #endif @@ -5989,7 +6988,10 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) @@ -5998,12 +7000,21 @@ Process *schedule(Process *p, int calls) } } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->check_children) { - esdp->check_children = 0; - erts_smp_runq_unlock(rq); - erts_check_children(); - erts_smp_runq_lock(rq); +#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) + { + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + long aux_work = erts_smp_atomic_read(&ssi->aux_work); + if (aux_work) { + erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + erts_smp_runq_lock(rq); + } } #endif @@ -6035,7 +7046,10 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; } @@ -6052,17 +7066,7 @@ Process *schedule(Process *p, int calls) } } - if (prepare_for_sys_schedule()) { - erts_smp_atomic_set(&function_calls, 0); - fcalls = 0; - sched_sys_wait(esdp->no, rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - else { - /* If all schedulers are waiting, one of them *should* - be waiting in erl_sys_schedule() */ - sched_cnd_wait(esdp->no, rq); - } + scheduler_wait(&fcalls, esdp, rq); non_empty_runq(rq); @@ -6124,7 +7128,7 @@ Process *schedule(Process *p, int calls) else { if (erts_common_run_queue) { if (erts_common_run_queue->waiting) - wake_one_scheduler(); + wake_scheduler(erts_common_run_queue, 0, 1); } else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); @@ -6439,8 +7443,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; - smp_notify_inc_runq(rq); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } static void @@ -6682,7 +7686,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq; + ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ #ifndef HYBRID @@ -6999,10 +8003,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif p->status = P_WAITING; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(notify_runq); + res = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 8f9f7f004e..6d7de237f3 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -89,6 +89,7 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP +extern Uint erts_max_main_threads; #include "erl_bits.h" #endif @@ -219,6 +220,51 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; +#define ERTS_SSI_FLG_SLEEPING (((long) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((long) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((long) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((long) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((long) 1) << 4) + +#define ERTS_SSI_FLGS_SLEEP_TYPE \ + (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) + +#define ERTS_SSI_FLGS_SLEEP \ + (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE) + +#define ERTS_SSI_FLGS_ALL \ + (ERTS_SSI_FLGS_SLEEP \ + | ERTS_SSI_FLG_WAITING \ + | ERTS_SSI_FLG_SUSPENDED) + + +#if !defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + && defined(ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN) +#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +#endif + +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((long) 1) << 0) + +#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN) +#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ + (0) + +typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; + +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; + +struct ErtsSchedulerSleepInfo_ { + ErtsSchedulerSleepInfo *next; + ErtsSchedulerSleepInfo *prev; + erts_smp_atomic_t flags; + erts_tse_t *event; + erts_smp_atomic_t aux_work; +}; + /* times to reschedule low prio process before running */ #define RESCHEDULE_LOW 8 @@ -271,8 +317,9 @@ struct ErtsRunQueue_ { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - erts_smp_atomic_t spin_waiter; - erts_smp_atomic_t spin_wake; +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -353,6 +400,7 @@ struct ErtsSchedulerData_ { ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ + ErtsSchedulerSleepInfo *ssi; Process *free_process; #endif #if !HEAP_ON_C_STACK @@ -374,11 +422,6 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; /* run queue mutex */ - int blocked_check_children; /* schdlr_sspnd mutex */ -#endif - erts_smp_atomic_t suspended; /* Only used when common run queue */ erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ #endif }; @@ -1085,6 +1128,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); void erts_deep_process_dump(int, void *); +Eterm erts_get_reader_groups_map(Process *c_p); +Eterm erts_debug_reader_groups_map(Process *c_p, int groups); + Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); @@ -1509,29 +1555,30 @@ extern int erts_disable_proc_not_running_opt; #define ERTS_MIN_PROCESSES 16 #endif -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_smp_notify_inc_runq__(ErtsRunQueue *runq); -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ +void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#ifdef ERTS_SMP +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, long); +ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (runq->waiting) - erts_smp_notify_inc_runq__(runq); -#endif + long flags = erts_smp_atomic_read(&ssi->flags); + ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) + || (flags & ERTS_SSI_FLG_WAITING)); + if (flags & ERTS_SSI_FLG_SLEEPING) { + flags = erts_smp_atomic_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + erts_sched_finish_poke(ssi, flags); + } } -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #ifdef ERTS_SMP */ + #include "erl_process_lock.h" #undef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 52440fb635..a4d12139e9 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -71,9 +71,12 @@ const Process erts_proc_lock_busy; #ifdef ERTS_SMP -/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/ -#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 16000 +#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 +#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 +#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50 + +#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25 #ifdef ERTS_PROC_LOCK_DEBUG #define ERTS_PROC_LOCK_HARD_DEBUG @@ -83,32 +86,19 @@ const Process erts_proc_lock_busy; static void check_queue(erts_proc_lock_t *lck); #endif - -typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t; -struct erts_proc_lock_waiter_t_ { - erts_proc_lock_waiter_t *next; - erts_proc_lock_waiter_t *prev; - ErtsProcLocks wait_locks; - erts_smp_gate_t gate; - erts_proc_lock_queues_t *queues; -}; +#if SIZEOF_INT < 4 +#error "The size of the 'uflgs' field of the erts_tse_t type is too small" +#endif struct erts_proc_lock_queues_t_ { erts_proc_lock_queues_t *next; - erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -}; - -struct erts_proc_lock_thr_spec_data_t_ { - erts_proc_lock_queues_t *qs; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; }; static erts_proc_lock_queues_t zeroqs = {0}; -static erts_smp_spinlock_t wtr_lock; -static erts_proc_lock_waiter_t *waiter_free_list; +static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; -static erts_tsd_key_t waiter_key; #ifdef ERTS_ENABLE_LOCK_CHECK static struct { @@ -122,35 +112,26 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; static int proc_lock_spin_count; -static int proc_lock_trans_spin_cost; +static int aux_thr_proc_lock_spin_count; -static void cleanup_waiter(void); +static void cleanup_tse(void); void erts_init_proc_lock(void) { int i; int cpus; - erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc"); + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#if ERTS_PROC_LOCK_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i)); -#else - erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif -#else #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i)); + erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif -#endif } - waiter_free_list = NULL; queue_free_list = NULL; - erts_tsd_key_create(&waiter_key); - erts_thr_install_exit_handler(cleanup_waiter); + erts_thr_install_exit_handler(cleanup_tse); #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); @@ -158,86 +139,106 @@ erts_init_proc_lock(void) lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif cpus = erts_get_cpu_configured(erts_cpuinfo); - if (cpus > 1) - proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE - * ((int) erts_no_schedulers)); - else if (cpus == 1) - proc_lock_spin_count = 0; - else /* No of cpus unknown. Assume multi proc, but be conservative. */ + if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; - if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) - proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; - proc_lock_trans_spin_cost = proc_lock_spin_count/20; -} - -static ERTS_INLINE erts_proc_lock_waiter_t * -alloc_wtr(void) -{ - erts_proc_lock_waiter_t *wtr; - erts_smp_spin_lock(&wtr_lock); - wtr = waiter_free_list; - if (wtr) { - waiter_free_list = wtr->next; - ERTS_LC_ASSERT(queue_free_list); - wtr->queues = queue_free_list; - queue_free_list = wtr->queues->next; - erts_smp_spin_unlock(&wtr_lock); + proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC + * ((int) erts_no_schedulers)); + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT; } - else { - erts_smp_spin_unlock(&wtr_lock); - wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR, - sizeof(erts_proc_lock_waiter_t)); - erts_smp_gate_init(&wtr->gate); - wtr->wait_locks = (ErtsProcLocks) 0; - wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, - sizeof(erts_proc_lock_queues_t)); - sys_memcpy((void *) wtr->queues, - (void *) &zeroqs, - sizeof(erts_proc_lock_queues_t)); + else if (cpus == 1) { + proc_lock_spin_count = 0; + aux_thr_proc_lock_spin_count = 0; } - return wtr; + else { /* No of cpus unknown. Assume multi proc, but be conservative. */ + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2; + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2; + } + if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; } #ifdef ERTS_ENABLE_LOCK_CHECK static void -check_unused_waiter(erts_proc_lock_waiter_t *wtr) +check_unused_tse(erts_tse_t *wtr) { int i; - ERTS_LC_ASSERT(wtr->wait_locks == 0); + erts_proc_lock_queues_t *queues = wtr->udata; + ERTS_LC_ASSERT(wtr->uflgs == 0); for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - ERTS_LC_ASSERT(!wtr->queues->queue[i]); + ERTS_LC_ASSERT(!queues->queue[i]); } -#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W)) +#define CHECK_UNUSED_TSE(W) check_unused_tse((W)) #else -#define CHECK_UNUSED_WAITER(W) +#define CHECK_UNUSED_TSE(W) #endif +static ERTS_INLINE erts_tse_t * +tse_fetch(erts_pix_lock_t *pix_lock) +{ + erts_tse_t *tse = erts_tse_fetch(); + if (!tse->udata) { + erts_proc_lock_queues_t *qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_unlock(pix_lock); +#endif + erts_smp_spin_lock(&qs_lock); + qs = queue_free_list; + if (qs) { + queue_free_list = queue_free_list->next; + erts_smp_spin_unlock(&qs_lock); + } + else { + erts_smp_spin_unlock(&qs_lock); + qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, + sizeof(erts_proc_lock_queues_t)); + sys_memcpy((void *) qs, + (void *) &zeroqs, + sizeof(erts_proc_lock_queues_t)); + } + tse->udata = qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_lock(pix_lock); +#endif + } + tse->uflgs = 0; + return tse; +} static ERTS_INLINE void -free_wtr(erts_proc_lock_waiter_t *wtr) +tse_return(erts_tse_t *tse, int force_free_q) { - CHECK_UNUSED_WAITER(wtr); - erts_smp_spin_lock(&wtr_lock); - wtr->next = waiter_free_list; - waiter_free_list = wtr; - wtr->queues->next = queue_free_list; - queue_free_list = wtr->queues; - erts_smp_spin_unlock(&wtr_lock); + CHECK_UNUSED_TSE(tse); + if (force_free_q || erts_tse_is_tmp(tse)) { + erts_proc_lock_queues_t *qs = tse->udata; + ASSERT(qs); + erts_smp_spin_lock(&qs_lock); + qs->next = queue_free_list; + queue_free_list = qs; + erts_smp_spin_unlock(&qs_lock); + tse->udata = NULL; + } + erts_tse_return(tse); } void erts_proc_lock_prepare_proc_lock_waiter(void) { - erts_tsd_set(waiter_key, (void *) alloc_wtr()); + tse_return(tse_fetch(NULL), 0); } static void -cleanup_waiter(void) +cleanup_tse(void) { - erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key); - if (wtr) - free_wtr(wtr); + erts_tse_t *tse = erts_tse_fetch(); + if (tse) { + if (tse->udata) + tse_return(tse, 1); + else + erts_tse_return(tse); + } } @@ -250,7 +251,7 @@ cleanup_waiter(void) static ERTS_INLINE void enqueue_waiter(erts_proc_lock_queues_t *qs, int ix, - erts_proc_lock_waiter_t *wtr) + erts_tse_t *wtr) { if (!qs->queue[ix]) { qs->queue[ix] = wtr; @@ -266,10 +267,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs, } } -static erts_proc_lock_waiter_t * +static erts_tse_t * dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) { - erts_proc_lock_waiter_t *wtr = qs->queue[ix]; + erts_tse_t *wtr = qs->queue[ix]; ERTS_LC_ASSERT(qs->queue[ix]); if (wtr->next == wtr) { ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr); @@ -295,10 +296,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) * lock. */ static ERTS_INLINE void -try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) +try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; - ErtsProcLocks locks = wtr->wait_locks; + ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); @@ -334,7 +335,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) } } - wtr->wait_locks &= ~got_locks; + wtr->uflgs &= ~got_locks; } /* @@ -350,8 +351,8 @@ transfer_locks(Process *p, int unlock) { int transferred = 0; - erts_proc_lock_waiter_t *wake = NULL; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wake = NULL; + erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; @@ -377,11 +378,11 @@ transfer_locks(Process *p, ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; - ERTS_LC_ASSERT(wtr->wait_locks & lock); - wtr->wait_locks &= ~lock; - if (wtr->wait_locks) + ERTS_LC_ASSERT(wtr->uflgs & lock); + wtr->uflgs &= ~lock; + if (wtr->uflgs) try_aquire(&p->lock, wtr); - if (!wtr->wait_locks) { + if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. @@ -412,9 +413,10 @@ transfer_locks(Process *p, erts_pix_unlock(pix_lock); do { - erts_proc_lock_waiter_t *tmp = wake; + erts_tse_t *tmp = wake; wake = wake->next; - erts_smp_gate_let_through(&tmp->gate, 1); + erts_atomic_set(&tmp->uaflgs, 0); + erts_tse_set(tmp); } while (wake); if (!unlock) @@ -462,26 +464,16 @@ wait_for_locks(Process *p, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); - int tsd; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; + erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ - wtr = erts_tsd_get(waiter_key); - if (wtr) - tsd = 1; - else { -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_unlock(pix_lock); -#endif - wtr = alloc_wtr(); - tsd = 0; -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_lock(pix_lock); -#endif - } + wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ - wtr->wait_locks = need_locks; + wtr->uflgs = need_locks; + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -489,14 +481,16 @@ wait_for_locks(Process *p, ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); + qs = wtr->udata; + ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { - wtr->queues->next = NULL; - p->lock.queues = wtr->queues; + qs->next = NULL; + p->lock.queues = qs; } else { - wtr->queues->next = p->lock.queues->next; - p->lock.queues->next = wtr->queues; + qs->next = p->lock.queues->next; + p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG @@ -506,46 +500,59 @@ wait_for_locks(Process *p, /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif - if (wtr->wait_locks) { /* We didn't get them all; need to wait... */ - /* Got to wait for locks... */ + if (wtr->uflgs) { + /* We didn't get them all; need to wait... */ + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + + erts_atomic_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); - /* - * Wait for needed locks. When we return all needed locks have - * have been acquired by other threads and transfered to us. - */ -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count); -#else - erts_smp_gate_wait(&wtr->gate); -#endif + while (1) { + int res; + erts_tse_reset(wtr); + + if (erts_atomic_read(&wtr->uaflgs) == 0) + break; + + /* + * Wait for needed locks. When we are woken all needed locks have + * have been acquired by other threads and transfered to us. + * However, we need to be prepared for spurious wakeups. + */ + do { + res = erts_tse_wait(wtr); /* might return EINTR */ + } while (res != 0); + } erts_pix_lock(pix_lock); + + ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { - wtr->queues = p->lock.queues->next; - p->lock.queues->next = wtr->queues->next; + qs = p->lock.queues->next; + p->lock.queues->next = qs->next; } else { - wtr->queues = p->lock.queues; + qs = p->lock.queues; p->lock.queues = NULL; } + wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); - if (tsd) - CHECK_UNUSED_WAITER(wtr); - else - free_wtr(wtr); + tse_return(wtr, 0); } /* @@ -563,52 +570,57 @@ erts_proc_lock_failed(Process *p, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - int spin_count = 0; -#else - int spin_count = proc_lock_spin_count; -#endif - + int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + int thr_spin_count; + int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; - while (need_locks != 0) - { - ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks); + if (erts_thr_get_main_status()) + thr_spin_count = proc_lock_spin_count; + else + thr_spin_count = aux_thr_proc_lock_spin_count; + + spin_count = thr_spin_count; + + while (need_locks != 0) { + ErtsProcLocks can_grab; + + can_grab = in_order_locks(olflgs, need_locks); - if (can_grab == 0) - { + if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ - if (spin_count-- <= 0) - { + if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } + ERTS_SPIN_BODY; + + if (--until_yield == 0) { + until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } - else - { + else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); - if (nflgs == olflgs) - { + if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; -#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE /* Since we made progress, reset the spin count. */ - spin_count = proc_lock_spin_count; -#endif + spin_count = thr_spin_count; } - else - { + else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } @@ -1407,7 +1419,7 @@ check_queue(erts_proc_lock_t *lck) wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index d71e5a0a6e..7cfc9893fa 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_t mtx; -#else erts_smp_spinlock_t spnlck; -#endif char buf[64]; /* Try to get locks in different cache lines */ } u; } erts_pix_lock_t; @@ -277,9 +273,12 @@ typedef struct { ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_acqb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic_cmpxchg_relb(&(L)->flags, \ + (long) (NEW), (long) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) @@ -289,6 +288,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *, ErtsProcLocks); ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *, ErtsProcLocks); +ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *, + ErtsProcLocks, + ErtsProcLocks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -322,7 +324,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) @@ -348,9 +352,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, - erts_pix_lock_t *, - ErtsProcLocks, - char *file, unsigned int line); + erts_pix_lock_t *, + ErtsProcLocks, + char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, erts_pix_lock_t *, @@ -372,30 +376,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_lock(&pixlck->u.mtx); -#else erts_smp_spin_lock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_unlock(&pixlck->u.mtx); -#else erts_smp_spin_unlock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { -#if ERTS_PROC_LOCK_MUTEX_IMPL - return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx); -#else return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); -#endif } /* @@ -417,9 +409,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) ErtsProcLocks expct_lflgs = 0; while (1) { - ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, - expct_lflgs | locks, - expct_lflgs); + ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, + expct_lflgs | locks, + expct_lflgs); if (ERTS_LIKELY(lflgs == expct_lflgs)) { /* We successfully grabbed all locks. */ return 0; @@ -535,7 +527,7 @@ erts_smp_proc_unlock__(Process *p, if (want_lflgs != old_lflgs) { ErtsProcLocks new_lflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs); if (new_lflgs != old_lflgs) { /* cmpxchg failed, try again. */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 03d2a586e3..b41fa70476 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -43,9 +43,17 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t; typedef erts_tid_t erts_smp_tid_t; typedef erts_mtx_t erts_smp_mtx_t; typedef erts_cnd_t erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER +#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED +#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED +typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_gate_t erts_smp_gate_t; typedef ethr_atomic_t erts_smp_atomic_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; @@ -54,15 +62,27 @@ void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0 +#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} typedef int erts_smp_thr_opts_t; typedef int erts_smp_thr_init_data_t; typedef int erts_smp_tid_t; typedef int erts_smp_mtx_t; typedef int erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_LONG_LIVED 0 +#define ERTS_SMP_RWMTX_SHORT_LIVED 0 +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef int erts_smp_gate_t; typedef long erts_smp_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; @@ -103,8 +123,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); @@ -119,9 +137,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); +ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); @@ -155,6 +181,16 @@ ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -190,12 +226,6 @@ ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount); #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 @@ -331,22 +361,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_set_forksafe(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unset_forksafe(mtx); -#endif -} - ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) { @@ -433,6 +447,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) } ERTS_GLB_INLINE void +erts_smp_rwmtx_set_reader_group(int no) +{ +#ifdef ERTS_SMP + erts_rwmtx_set_reader_group(no); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -441,6 +474,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt(rwmtx, opt, name); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) { #ifdef ERTS_SMP @@ -709,6 +752,73 @@ erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) #endif } +ERTS_GLB_INLINE long +erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i) +{ +#ifdef ERTS_SMP + erts_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { @@ -919,54 +1029,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_smp_gate_init(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_init((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_destroy(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_destroy((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_close(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_close((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no) -{ -#ifdef ERTS_SMP - erts_gate_let_through((erts_gate_t *) gp, no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_wait(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_wait((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount) -{ -#ifdef ERTS_SMP - erts_gate_swait((erts_gate_t *) gp, spincount); -#endif -} - #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 35b338c6eb..0b7269262e 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -25,6 +25,11 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +#define ERTS_SPIN_BODY ETHR_SPIN_BODY + +#define ERTS_MAX_READER_GROUPS 8 +extern int erts_reader_groups; + #include "sys.h" #ifdef USE_THREADS @@ -48,6 +53,7 @@ #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; +typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ @@ -73,8 +79,19 @@ typedef struct { erts_lcnt_lock_t lcnt; #endif } erts_rwmtx_t; + +#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER +#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL +#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED +#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED +#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED +typedef ethr_rwmutex_opt erts_rwmtx_opt_t; + typedef ethr_tsd_key erts_tsd_key_t; -typedef ethr_gate erts_gate_t; +typedef ethr_ts_event erts_tse_t; typedef ethr_atomic_t erts_atomic_t; /* spinlock */ @@ -103,25 +120,14 @@ typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_REC_MTX_INITER \ - {ETHR_REC_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)} -#define ERTS_MTX_INITER \ - {ETHR_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)} -#else -#define ERTS_REC_MTX_INITER {ETHR_REC_MUTEX_INITER} -#define ERTS_MTX_INITER {ETHR_MUTEX_INITER} -#endif -#define ERTS_CND_INITER ETHR_COND_INITER #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER +#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ + ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif - #else /* #ifdef USE_THREADS */ #define ERTS_THR_MEMORY_BARRIER @@ -129,12 +135,26 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; typedef int erts_thr_init_data_t; +typedef int erts_thr_late_init_data_t; typedef int erts_tid_t; typedef int erts_mtx_t; typedef int erts_cnd_t; +#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_RWMTX_TYPE_NORMAL 0 +#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_RWMTX_LONG_LIVED 0 +#define ERTS_RWMTX_SHORT_LIVED 0 +#define ERTS_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_rwmtx_opt_t; typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; -typedef int erts_gate_t; +typedef int erts_tse_t; typedef long erts_atomic_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -148,7 +168,6 @@ typedef struct { long tv_nsec; } erts_thr_timeval_t; -#define ERTS_REC_MTX_INITER 0 #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -158,6 +177,7 @@ typedef struct { #endif /* #ifdef USE_THREADS */ ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); +ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); @@ -166,9 +186,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx); -#endif ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, @@ -177,8 +194,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); @@ -192,9 +207,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); +ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); @@ -228,6 +251,20 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long expected); ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); +ERTS_GLB_INLINE long erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, long i); +ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp); +ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, + char *name, + Eterm extra, + Uint16 opt); ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra); @@ -263,12 +300,16 @@ ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); -ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount); +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); +ERTS_GLB_INLINE int erts_thr_get_main_status(void); +ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 @@ -290,6 +331,16 @@ erts_thr_init(erts_thr_init_data_t *id) } ERTS_GLB_INLINE void +erts_thr_late_init(erts_thr_late_init_data_t *id) +{ +#ifdef USE_THREADS + int res = ethr_late_init(id); + if (res) + erts_thr_fatal_error(res, "complete initialization of thread library"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { @@ -362,20 +413,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) #endif } - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_rec_mtx_init(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_rec_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize recursive mutex"); -#endif -} -#endif - - ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { @@ -422,9 +459,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -463,9 +498,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -492,26 +525,6 @@ erts_mtx_destroy(erts_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_mtx_set_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_set_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "set mutex forksafe"); -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_unset_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_unset_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "unset mutex forksafe"); -#endif -} - ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx) { @@ -531,11 +544,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); -#endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try lock mutex"); - +#endif return res; #else return 0; @@ -551,19 +560,16 @@ erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif - res = ethr_mutex_lock(&mtx->mtx); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "lock mutex"); #endif } @@ -571,16 +577,13 @@ ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif - res = ethr_mutex_unlock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "unlock mutex"); + ethr_mutex_unlock(&mtx->mtx); #endif } @@ -648,9 +651,7 @@ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_signal(cnd); - if (res) - erts_thr_fatal_error(res, "signal on condition variable"); + ethr_cond_signal(cnd); #endif } @@ -659,19 +660,34 @@ ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_broadcast(cnd); - if (res) - erts_thr_fatal_error(res, "broadcast on condition variable"); + ethr_cond_broadcast(cnd); #endif } /* rwmutex */ ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) +erts_rwmtx_set_reader_group(int no) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); +#endif + res = ethr_rwmutex_set_reader_group(no); + if (res != 0) + erts_thr_fatal_error(res, "set reader group"); +#endif +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef USE_THREADS + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -684,10 +700,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, + char *name, + Eterm extra) +{ + erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -700,6 +726,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) } ERTS_GLB_INLINE void +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +{ + erts_rwmtx_init_opt(rwmtx, NULL, name); +} + +ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS @@ -736,9 +768,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try read lock rwmutex"); return res; #else @@ -754,19 +783,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_rlock(&rwmtx->rwmtx); + ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "read lock rwmutex"); #endif } @@ -774,16 +800,13 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_runlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "read unlock rwmutex"); + ethr_rwmutex_runlock(&rwmtx->rwmtx); #endif } @@ -808,9 +831,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try write lock rwmutex"); return res; #else @@ -826,19 +846,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwlock(&rwmtx->rwmtx); + ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "write lock rwmutex"); #endif } @@ -846,16 +863,13 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "write unlock rwmutex"); + ethr_rwmutex_rwunlock(&rwmtx->rwmtx); #endif } @@ -917,9 +931,7 @@ ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_init(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic init"); + ethr_atomic_init(var, i); #else *var = i; #endif @@ -929,9 +941,7 @@ ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i) { #ifdef USE_THREADS - int res = ethr_atomic_set(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic set"); + ethr_atomic_set(var, i); #else *var = i; #endif @@ -941,11 +951,7 @@ ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS - long i; - int res = ethr_atomic_read(var, &i); - if (res) - erts_thr_fatal_error(res, "perform atomic read"); - return i; + return ethr_atomic_read(var); #else return *var; #endif @@ -955,11 +961,7 @@ ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_inctest(incp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic increment and test"); - return test; + return ethr_atomic_inc_read(incp); #else return ++(*incp); #endif @@ -969,11 +971,7 @@ ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_dectest(decp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement and test"); - return test; + return ethr_atomic_dec_read(decp); #else return --(*decp); #endif @@ -983,9 +981,7 @@ ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp) { #ifdef USE_THREADS - int res = ethr_atomic_inc(incp); - if (res) - erts_thr_fatal_error(res, "perform atomic increment"); + ethr_atomic_inc(incp); #else ++(*incp); #endif @@ -995,9 +991,7 @@ ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp) { #ifdef USE_THREADS - int res = ethr_atomic_dec(decp); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement"); + ethr_atomic_dec(decp); #else --(*decp); #endif @@ -1007,11 +1001,7 @@ ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_addtest(addp, i, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic addition and test"); - return test; + return ethr_atomic_add_read(addp, i); #else return *addp += i; #endif @@ -1021,9 +1011,7 @@ ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i) { #ifdef USE_THREADS - int res = ethr_atomic_add(addp, i); - if (res) - erts_thr_fatal_error(res, "perform atomic addition"); + ethr_atomic_add(addp, i); #else *addp += i; #endif @@ -1034,9 +1022,7 @@ erts_atomic_xchg(erts_atomic_t *xchgp, long new) { long old; #ifdef USE_THREADS - int res = ethr_atomic_xchg(xchgp, new, &old); - if (res) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_xchg(xchgp, new); #else old = *xchgp; *xchgp = new; @@ -1048,11 +1034,7 @@ ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) { #ifdef USE_THREADS - long old; - int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old); - if (ERTS_UNLIKELY(res != 0)) - erts_thr_fatal_error(res, "perform atomic exchange"); - return old; + return ethr_atomic_cmpxchg(xchgp, new, expected); #else long old = *xchgp; if (old == expected) @@ -1064,31 +1046,95 @@ erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_or_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise or"); + return ethr_atomic_read_bor(var, mask); #else + long old; old = *var; *var |= mask; -#endif return old; +#endif } ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_and_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise and"); + return ethr_atomic_read_band(var, mask); #else + long old; old = *var; *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE long +erts_atomic_read_acqb(erts_atomic_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_set_relb(erts_atomic_t *var, long i) +{ +#ifdef USE_THREADS + ethr_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_dec_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE long +erts_atomic_dectest_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic_dec_read_relb(decp); +#else + return --(*decp); #endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + long new, + long exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_relb(xchgp, new, exp); +#else + long old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif } /* spinlock */ @@ -1112,6 +1158,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, + Uint16 opt) +{ +#ifdef USE_THREADS + int res = ethr_spinlock_init(&lock->slck); + if (res) + erts_thr_fatal_error(res, "init spinlock"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); +#endif +#else + (void)lock; +#endif +} + + +ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name) { #ifdef USE_THREADS @@ -1152,16 +1218,13 @@ ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif - res = ethr_spin_unlock(&lock->slck); - if (res) - erts_thr_fatal_error(res, "release spin lock"); + ethr_spin_unlock(&lock->slck); #else (void)lock; #endif @@ -1175,19 +1238,16 @@ erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif - res = ethr_spin_lock(&lock->slck); + ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take spin lock"); #else (void)lock; #endif @@ -1268,16 +1328,13 @@ ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release read lock"); + ethr_read_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1291,19 +1348,16 @@ erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_lock(&lock->rwlck); + ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take read lock"); #else (void)lock; #endif @@ -1313,16 +1367,13 @@ ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release write lock"); + ethr_write_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1336,19 +1387,16 @@ erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_lock(&lock->rwlck); + ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take write lock"); #else (void)lock; #endif @@ -1432,66 +1480,95 @@ erts_tsd_get(erts_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_gate_init(erts_gate_t *gp) +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { #ifdef USE_THREADS - int res = ethr_gate_init((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "initialize gate"); + return (erts_tse_t *) ethr_get_ts_event(); +#else + return (erts_tse_t *) NULL; #endif } -ERTS_GLB_INLINE void -erts_gate_destroy(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_destroy((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "destroy gate"); + ethr_leave_ts_event(ep); #endif } -ERTS_GLB_INLINE void -erts_gate_close(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_close((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "close gate"); + ethr_event_set(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_let_through(erts_gate_t *gp, unsigned no) +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_let_through((ethr_gate *) gp, no); - if (res != 0) - erts_thr_fatal_error(res, "let through gate"); + ethr_event_reset(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_wait(erts_gate_t *gp) +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return ethr_event_wait(&((ethr_ts_event *) ep)->event); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) +{ +#ifdef USE_THREADS + return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_wait((ethr_gate *) gp); + return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) +{ +#ifdef USE_THREADS + int res = ethr_set_main_thr_status(on, no); if (res != 0) - erts_thr_fatal_error(res, "wait on gate"); + erts_thr_fatal_error(res, "set thread main status"); #endif } -ERTS_GLB_INLINE void -erts_gate_swait(erts_gate_t *gp, int spincount) +ERTS_GLB_INLINE int erts_thr_get_main_status(void) { #ifdef USE_THREADS - int res = ethr_gate_swait((ethr_gate *) gp, spincount); + int main_status; + int res = ethr_get_main_thr_status(&main_status); if (res != 0) - erts_thr_fatal_error(res, "swait on gate"); + erts_thr_fatal_error(res, "get thread main status"); + return main_status; +#else + return 1; #endif } +ERTS_GLB_INLINE void erts_thr_yield(void) +{ +#ifdef USE_THREADS + int res = ETHR_YIELD(); + if (res != 0) + erts_thr_fatal_error(res, "yield"); +#endif +} + + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 66b05c0e9d..5e81a2d624 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -43,8 +43,6 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. #define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) #define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) #define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) -#define export_init_lock() erts_smp_rwmtx_init(&export_table_lock, \ - "export_tab") extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; @@ -111,8 +109,12 @@ void init_export_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); - export_init_lock(); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 900ebcbbf7..c9bb7bbe91 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -39,8 +39,6 @@ static Hash process_reg; static erts_smp_rwmtx_t regtab_rwmtx; -#define reg_lock_init() erts_smp_rwmtx_init(®tab_rwmtx, \ - "reg_tab") #define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) #define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) #define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) @@ -147,8 +145,11 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - reg_lock_init(); + erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 737ffd9f94..400ef6c0ce 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -384,18 +384,6 @@ MALLOC_USE_HASH(1); #endif #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} #ifdef ERTS_THR_HAVE_SIG_FUNCS /* @@ -488,9 +476,6 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -538,13 +523,14 @@ erts_sys_pre_init(void) #endif #endif /* USE_THREADS */ erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); } void erl_sys_init(void) { + erts_smp_rwmtx_init(&environ_rwmtx, "environ"); #if !DISABLE_VFORK + { int res; char bindir[MAXPATHLEN]; size_t bindirsz = sizeof(bindir); @@ -574,6 +560,7 @@ erl_sys_init(void) bindir, DIR_SEPARATOR_CHAR, CHILD_SETUP_PROG_NAME); + } #endif #ifdef USE_SETLINEBUF diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index bd02cf85eb..54f71b202d 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -2974,19 +2974,6 @@ check_supported_os_version(void) } #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} - #ifdef ERTS_ENABLE_LOCK_COUNT static void thr_create_prepare_child(void *vtcdp) @@ -3005,14 +2992,9 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; - #ifdef ERTS_ENABLE_LOCK_COUNT eid.thread_create_child_func = thr_create_prepare_child; #endif - erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init(); diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index ab727b7cb1..4f4458802c 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -33,7 +33,7 @@ -include("test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). -export([equal/1, few_low/1, @@ -49,7 +49,8 @@ cpu_topology/1, sct_cmd/1, sbt_cmd/1, - scheduler_suspend/1]). + scheduler_suspend/1, + reader_groups/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(10)). @@ -67,7 +68,8 @@ all(suite) -> equal_with_high_max, bound_process, scheduler_bind, - scheduler_suspend]. + scheduler_suspend, + reader_groups]. init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), @@ -81,6 +83,10 @@ fin_per_testcase(_Case, Config) when is_list(Config) -> ?t:timetrap_cancel(Dog), ok. +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)). -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000). @@ -965,12 +971,361 @@ sst3_loop(S, N) -> erlang:system_flag(schedulers_online, 1), erlang:system_flag(schedulers_online, S), sst3_loop(S, N-1). + +reader_groups(Config) when is_list(Config) -> + %% White box testing. These results are correct, but other results + %% could be too... + + %% The actual tilepro64 topology + CPUT0 = [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}], + + ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1}, + {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2}, + {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4}, + {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5}, + {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5}, + {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6}, + {58,8},{60,6},{61,6}] + = reader_groups_map(CPUT0, 8), + + CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]), + c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]), + c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]), + c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]), + p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]), + c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]), + c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]), + c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]), + n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]), + c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]), + c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]), + c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]), + p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]), + c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]), + c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]), + c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]), + n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]), + c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]), + c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]), + c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]), + p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]), + c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]), + c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]), + c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]), + n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]), + c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]), + c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]), + c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]), + p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]), + c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]), + c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]), + c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3}, + {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5}, + {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7}, + {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10}, + {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12}, + {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14}, + {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16}, + {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18}, + {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20}, + {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22}, + {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24}, + {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26}, + {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27}, + {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29}, + {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31}, + {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}] + = reader_groups_map(CPUT1, 128), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1}, + {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1}, + {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1}, + {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1}, + {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1}, + {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2}, + {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2}, + {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2}, + {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2}, + {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2}, + {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2}, + {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2}, + {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2}, + {125,2},{126,2},{127,2}] + = reader_groups_map(CPUT1, 2), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3}, + {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4}, + {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5}, + {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6}, + {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7}, + {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8}, + {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10}, + {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11}, + {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12}, + {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13}, + {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14}, + {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14}, + {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15}, + {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16}, + {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17}, + {125,17},{126,17},{127,17}] + = reader_groups_map(CPUT1, 17), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2}, + {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2}, + {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3}, + {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3}, + {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4}, + {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5}, + {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5}, + {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6}, + {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6}, + {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7}, + {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7}, + {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7}, + {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7}, + {125,7},{126,7},{127,7}] + = reader_groups_map(CPUT1, 7), + + ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]), + p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2},{6,2},{7,2},{8,2},{9,2}, + {10,3}, + {11,4},{12,4},{13,4}, + {14,5},{15,5}] = reader_groups_map(CPUT2, 5), + + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,5},{13,5}, + {14,6},{15,6}] = reader_groups_map(CPUT2, 6), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,7}] = reader_groups_map(CPUT2, 7), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,8}] = reader_groups_map(CPUT2, 8), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,7}, + {14,8},{15,9}] = reader_groups_map(CPUT2, 9), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,8}, + {14,9},{15,10}] = reader_groups_map(CPUT2, 10), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,4}, + {5,5},{6,5},{7,5},{8,5},{9,5}, + {10,6}, + {11,7},{12,8},{13,9}, + {14,10},{15,11}] = reader_groups_map(CPUT2, 11), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,5}, + {5,6},{6,6},{7,6},{8,6},{9,6}, + {10,7}, + {11,8},{12,9},{13,10}, + {14,11},{15,12}] = reader_groups_map(CPUT2, 100), + + CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))]), + p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])], + + ?line [{0,5},{1,5},{2,6},{3,6},{4,6}, + {5,1},{6,1},{7,1},{8,1},{9,1}, + {10,2},{11,3},{12,3},{13,3}, + {14,4},{15,4}] = reader_groups_map(CPUT3, 6), + + CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]), + p([t(l(5))]), + p([c(l(6)),c(l(7)),c(l(8))]), + p([c(l(9)),c(l(10))]), + p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,3},{8,3}, + {9,4},{10,4}, + {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,4},{8,4}, + {9,5},{10,5}, + {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7), + + ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10), + + ?line ok. +reader_groups_map(CPUT, Groups) -> + Old = erlang:system_info({cpu_topology, defined}), + erlang:system_flag(cpu_topology, CPUT), + enable_internal_state(), + Res = erts_debug:get_internal_state({reader_groups_map, Groups}), + erlang:system_flag(cpu_topology, Old), + lists:sort(Res). + %% %% Utils %% +tilera_cpu_topology() -> + [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}]. + +l(Id) -> + {logical, Id}. + +t(X) -> + {thread, X}. + +c(X) -> + {core, X}. + +p(X) -> + {processor, X}. + +n(X) -> + {node, X}. + mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index f79f5cc978..76ce0a7e3c 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -131,6 +131,12 @@ static char *plush_val_switches[] = { NULL }; +/* +r arguments with values */ +static char *plusr_val_switches[] = { + "g", + NULL +}; + /* * Define sleep(seconds) in terms of Sleep() on Windows. @@ -872,6 +878,21 @@ int main(int argc, char **argv) i++; } break; + case 'r': + if (!is_one_of_strings(&argv[i][2], + plusr_val_switches)) + goto the_default; + else { + if (i+1 >= argc + || argv[i+1][0] == '-' + || argv[i+1][0] == '+') + usage(argv[i]); + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + } + break; case 's': if (!is_one_of_strings(&argv[i][2], pluss_val_switches)) @@ -1069,11 +1090,12 @@ usage_aux(void) "[-hybrid] " #endif "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] " - "[-args_file FILENAME] " - "[+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] " + "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] " + "[+h HEAP_SIZE_OPTION] [+K BOOLEAN] " "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+R COMPAT_REL] " - "[+r] [+s SCHEDULER_OPTION] [+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] [+W<i|w>] " - "[args ...]\n"); + "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] " + "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] " + "[+W<i|w>] [args ...]\n"); exit(1); } diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h index 82e9ba3798..6b875ff824 100644 --- a/erts/include/internal/erl_misc_utils.h +++ b/erts/include/internal/erl_misc_utils.h @@ -50,4 +50,8 @@ int erts_unbind_from_cpu_str(char *str); int erts_milli_sleep(long); +#ifdef __WIN32__ +int erts_get_last_win_errno(void); +#endif + #endif /* #ifndef ERL_MISC_UTILS_H_ */ diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h new file mode 100644 index 0000000000..e9c3daf783 --- /dev/null +++ b/erts/include/internal/ethr_internal.h @@ -0,0 +1,67 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Internal ethread exports + * Author: Rickard Green + */ + +#ifndef ETHR_INTERNAL_H__ +#define ETHR_INTERNAL_H__ + +#include "erl_misc_utils.h" + +extern ethr_memory_allocators ethr_mem__; +extern erts_cpu_info_t *ethr_cpu_info__; +extern size_t ethr_pagesize__; +extern size_t ethr_min_stack_size__; /* kilo words */ +extern size_t ethr_max_stack_size__; /* kilo words */ +extern int ethr_not_completely_inited__; +extern int ethr_not_inited__; + +extern void *(*ethr_thr_prepare_func__)(void); +extern void (*ethr_thr_parent_func__)(void *); +extern void (*ethr_thr_child_func__)(void *); + +#define ETHR_PAGE_ALIGN(SZ) \ + (((((size_t) (SZ)) - 1)/ethr_pagesize__ + 1)*ethr_pagesize__) +#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) +#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) + +#undef ETHR_STACK_GUARD_SIZE +#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE +# define ETHR_STACK_GUARD_SIZE (ethr_pagesize__) +#endif + +/* implemented in lib_src/<thr-lib>/ethread.c */ +int ethr_set_tse__(ethr_ts_event *tsep); +ethr_ts_event *ethr_get_tse__(void); +ETHR_PROTO_NORETURN__ ethr_abort__(void); +#ifdef ETHR_WIN32_THREADS +int ethr_win_get_errno__(void); +#endif + +/* implemented in lib_src/common/ethread_aux.c */ +int ethr_init_common__(ethr_init_data *id); +int ethr_late_init_common__(ethr_late_init_data *lid); +void ethr_run_exit_handlers__(void); +void ethr_ts_event_destructor__(void *vtsep); + + +#endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h new file mode 100644 index 0000000000..4ce3e75c78 --- /dev/null +++ b/erts/include/internal/ethr_mutex.h @@ -0,0 +1,510 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifndef ETHR_MUTEX_H__ +#define ETHR_MUTEX_H__ + +#define ETHR_RWMUTEX_INITIALIZED 0x99999999 +#define ETHR_MUTEX_INITIALIZED 0x77777777 +#define ETHR_COND_INITIALIZED 0x55555555 + +#if 0 +# define ETHR_MTX_HARD_DEBUG +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +# ifdef __GNUC__ +# warning ETHR_MTX_HARD_DEBUG +# endif +/*# define ETHR_MTX_HARD_DEBUG_LFS*/ +/*# define ETHR_MTX_HARD_DEBUG_FENCE*/ +/*# define ETHR_MTX_HARD_DEBUG_Q*/ +# define ETHR_MTX_HARD_DEBUG_WSQ + +# if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q) +# define ETHR_MTX_HARD_DEBUG_WSQ +# endif +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#if 0 +# define ETHR_MTX_Q_LOCK_SPINLOCK__ +# define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t +#elif defined(ETHR_PTHREADS) +# define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__ +# define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t +#elif defined(ETHR_WIN32_THREADS) +# define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__ +# define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION +#else +# error Need a qlock implementation +#endif + +#define ETHR_RWMTX_W_FLG__ (((long) 1) << 31) +#define ETHR_RWMTX_W_WAIT_FLG__ (((long) 1) << 30) +#define ETHR_RWMTX_R_WAIT_FLG__ (((long) 1) << 29) + +/* frequent read kind */ +#define ETHR_RWMTX_R_FLG__ (((long) 1) << 28) +#define ETHR_RWMTX_R_PEND_UNLCK_MASK__ (ETHR_RWMTX_R_FLG__ - 1) +#define ETHR_RWMTX_R_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) + +/* normal kind */ +#define ETHR_RWMTX_RS_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) + +#define ETHR_RWMTX_WAIT_FLGS__ \ + (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__) + +#define ETHR_CND_WAIT_FLG__ ETHR_RWMTX_R_WAIT_FLG__ + +struct ethr_mutex_base_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long pre_fence; +#endif + ethr_atomic_t flgs; + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; + short aux_scnt; + short main_scnt; +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + int ws; +#endif +#ifdef ETHR_MTX_HARD_DEBUG_LFS + ethr_atomic_t hdbg_lfs; +#endif +}; + +#endif + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_mutex_opt; + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_cond_opt; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + struct ethr_mutex_base_ mtxb; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + struct { + long pre_fence; + } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */ +#endif + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; + short aux_scnt; + short main_scnt; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread */ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + pthread_mutex_t pt_mtx; +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { + pthread_cond_t pt_cnd; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread */ + +int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *); +int ethr_mutex_init(ethr_mutex *); +int ethr_mutex_destroy(ethr_mutex *); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) +int ethr_mutex_trylock(ethr_mutex *); +void ethr_mutex_lock(ethr_mutex *); +void ethr_mutex_unlock(ethr_mutex *); +#endif +int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *); +int ethr_cond_init(ethr_cond *); +int ethr_cond_destroy(ethr_cond *); +void ethr_cond_signal(ethr_cond *); +void ethr_cond_broadcast(ethr_cond *); +int ethr_cond_wait(ethr_cond *, ethr_mutex *); + +typedef enum { + ETHR_RWMUTEX_TYPE_NORMAL, + ETHR_RWMUTEX_TYPE_FREQUENT_READ, + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +} ethr_rwmutex_type; + +typedef enum { + ETHR_RWMUTEX_LONG_LIVED, + ETHR_RWMUTEX_SHORT_LIVED, + ETHR_RWMUTEX_UNKNOWN_LIVED +} ethr_rwmutex_lived; + +typedef struct { + ethr_rwmutex_type type; + ethr_rwmutex_lived lived; + int main_spincount; + int aux_spincount; +} ethr_rwmutex_opt; + +#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \ + {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +typedef union { + struct { + ethr_atomic_t readers; + int waiting_readers; + int byte_offset; + ethr_rwmutex_lived lived; + } data; + char align__[ETHR_CACHE_LINE_SIZE]; +} ethr_rwmtx_readers_array__; + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + struct ethr_mutex_base_ mtxb; + ethr_rwmutex_type type; + ethr_ts_event *rq_end; + union { + ethr_rwmtx_readers_array__ *ra; + int rs; + } tdata; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread_rwlock */ + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + pthread_rwlock_t pt_rwlock; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread_rwlock */ + +int ethr_rwmutex_set_reader_group(int); +int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *); +int ethr_rwmutex_init(ethr_rwmutex *); +int ethr_rwmutex_destroy(ethr_rwmutex *); +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \ + || !defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_MUTEX_IMPL__) +int ethr_rwmutex_tryrlock(ethr_rwmutex *); +void ethr_rwmutex_rlock(ethr_rwmutex *); +void ethr_rwmutex_runlock(ethr_rwmutex *); +int ethr_rwmutex_tryrwlock(ethr_rwmutex *); +void ethr_rwmutex_rwlock(ethr_rwmutex *); +void ethr_rwmutex_rwunlock(ethr_rwmutex *); +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +#define ETHR_MTX_HARD_ASSERT(A) \ + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) +#else +#define ETHR_MTX_HARD_ASSERT(A) ((void) 1) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_LFS +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) \ +do { \ + ethr_atomic_init(&(MTXB)->hdbg_lfs, 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) \ +do { \ + long val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic_inc_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ > 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) \ +do { \ + long val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ >= 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) \ +do { \ + long val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == -1); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) \ +do { \ + long val__ = ethr_atomic_inctest(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#else +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeadbeefdeadbeefL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeaddeaddeadL +#else +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeaddeadL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeadL +#endif + +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\ + ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \ +do { \ + (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \ + (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \ +} while (0) +#else +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) +#endif + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50 + +#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0 +#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0 + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +void ethr_mutex_lock_wait__(ethr_mutex *, long); +void ethr_mutex_unlock_wake__(ethr_mutex *, long); + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + long act; + int res; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + res = (act == 0) ? 0 : EBUSY; + + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + ETHR_COMPILER_BARRIER; + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + long act; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + ethr_mutex_lock_wait__(mtx, act); + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + long act; + ETHR_COMPILER_BARRIER; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb); + + act = ethr_atomic_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + ethr_mutex_unlock_wake__(mtx, act); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#else /* pthread_mutex */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + int res; + res = pthread_mutex_trylock(&mtx->pt_mtx); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_lock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_unlock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_mutex */ + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50 + +#else /* pthread_rwlock */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_rwlock */ + +int ethr_mutex_lib_init(int); +int ethr_mutex_lib_late_init(int, int); + +#endif /* #ifndef ETHR_MUTEX_H__ */ diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h new file mode 100644 index 0000000000..2f9f987d0b --- /dev/null +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -0,0 +1,185 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: "Optimized" fallbacks used when native ops are missing + * Author: Rickard Green + */ + +#ifndef ETHR_OPTIMIZED_FALLBACKS_H__ +#define ETHR_OPTIMIZED_FALLBACKS_H__ + +#ifdef ETHR_HAVE_NATIVE_ATOMICS +#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 +#endif + +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 +#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) +/* --- Optimized spinlocks using pthread spinlocks -------------------------- */ +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +typedef pthread_spinlock_t ethr_opt_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_init((pthread_spinlock_t *) lock, 0); +} + +static ETHR_INLINE int +ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_destroy((pthread_spinlock_t *) lock); +} + + +static ETHR_INLINE int +ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_unlock((pthread_spinlock_t *) lock); +} + +static ETHR_INLINE int +ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_lock((pthread_spinlock_t *) lock); +} + +#endif + +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native spinlocks using native atomics -------------------------------- */ +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +typedef ethr_native_atomic_t ethr_native_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *lock) +{ + ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) == 1); + ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *lock) +{ + while (ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + (long) 1, (long) 0) != 0) { + ETHR_SPIN_BODY; + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#endif + + +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native rwspinlocks using native atomics ------------------------------ */ +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 + +typedef ethr_native_atomic_t ethr_native_rwlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +#define ETHR_WLOCK_FLAG__ (((long) 1) << 30) + +static ETHR_INLINE void +ethr_native_rwlock_init(ethr_native_rwlock_t *lock) +{ + ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_read_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; +#ifdef DEBUG + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) >= 0); +#endif + ethr_native_atomic_dec_relb((ethr_native_atomic_t *) lock); +} + +static ETHR_INLINE void +ethr_native_read_lock(ethr_native_rwlock_t *lock) +{ + long act, exp = 0; + while (1) { + act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + exp+1, exp); + if (act == exp) + break; + ETHR_SPIN_BODY; + exp = (act & ETHR_WLOCK_FLAG__) ? 0 : act; + } + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ethr_native_write_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) + == ETHR_WLOCK_FLAG__); + ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); +} + +static ETHR_INLINE void +ethr_native_write_lock(ethr_native_rwlock_t *lock) +{ + long act, exp = 0; + while (1) { + act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, + exp|ETHR_WLOCK_FLAG__, exp); + if (act == exp) + break; + ETHR_SPIN_BODY; + exp = act & ~ETHR_WLOCK_FLAG__; + } + act |= ETHR_WLOCK_FLAG__; + /* Wait for readers to leave */ + while (act != ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ethr_native_atomic_read_acqb((ethr_native_atomic_t *) lock); + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#endif + +#endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 4e7a38cd5c..4a205699bd 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -33,20 +33,9 @@ #include <stdlib.h> #include "erl_errno.h" -/* - * Extra memory barrier requirements: - * - ethr_atomic_or_old() needs to enforce a memory barrier sufficient - * for a lock operation. - * - ethr_atomic_and_old() needs to enforce a memory barrier sufficient - * for an unlock operation. - * - ethr_atomic_cmpxchg() needs to enforce a memory barrier sufficient - * for a lock and unlock operation. - */ - - -#undef ETHR_USE_RWMTX_FALLBACK #undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#undef ETHR_HAVE_OPTIMIZED_LOCKS +#undef ETHR_HAVE_OPTIMIZED_SPINLOCK +#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK typedef struct { long tv_sec; @@ -54,6 +43,10 @@ typedef struct { } ethr_timeval; #if defined(DEBUG) +# define ETHR_DEBUG +#endif + +#if defined(ETHR_DEBUG) # undef ETHR_XCHK # define ETHR_XCHK 1 #else @@ -68,47 +61,57 @@ typedef struct { #elif defined(__WIN32__) # define ETHR_INLINE __forceinline #endif -#if defined(DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ +#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC)) # undef ETHR_INLINE # define ETHR_INLINE # undef ETHR_TRY_INLINE_FUNCS #endif -#ifdef ETHR_FORCE_INLINE_FUNCS -# define ETHR_TRY_INLINE_FUNCS -#endif -#if !defined(ETHR_DISABLE_NATIVE_IMPLS) \ - && (defined(PURIFY) || defined(VALGRIND) || defined(ERTS_MIXED_CYGWIN_VC)) +#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && (defined(PURIFY)||defined(VALGRIND)) # define ETHR_DISABLE_NATIVE_IMPLS #endif -#define ETHR_RWMUTEX_INITIALIZED 0x99999999 -#define ETHR_MUTEX_INITIALIZED 0x77777777 -#define ETHR_COND_INITIALIZED 0x55555555 +/* Assume 64-byte cache line size */ +#define ETHR_CACHE_LINE_SIZE 64L +#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) -#define ETHR_CACHE_LINE_SIZE 64 +#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ + (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE) -#ifdef ETHR_INLINE_FUNC_NAME_ -# define ETHR_CUSTOM_INLINE_FUNC_NAME_ -#else +#ifndef ETHR_INLINE_FUNC_NAME_ # define ETHR_INLINE_FUNC_NAME_(X) X #endif -#define ETHR_COMPILER_BARRIER ethr_compiler_barrier() -#ifdef __GNUC__ -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER __asm__ __volatile__("":::"memory") +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif #endif -#ifdef DEBUG +int ethr_assert_failed(const char *file, int line, const char *func, char *a); +#ifdef ETHR_DEBUG #define ETHR_ASSERT(A) \ - ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) -int ethr_assert_failed(char *f, int l, char *a); + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A))) #else #define ETHR_ASSERT(A) ((void) 1) #endif +#if defined(__GNUC__) +# define ETHR_PROTO_NORETURN__ void __attribute__((noreturn)) +# define ETHR_IMPL_NORETURN__ void +#elif defined(__WIN32__) && defined(_MSC_VER) +# define ETHR_PROTO_NORETURN__ __declspec(noreturn) void +# define ETHR_IMPL_NORETURN__ __declspec(noreturn) void +#else +# define ETHR_PROTO_NORETURN__ void +# define ETHR_IMPL_NORETURN__ void +#endif + #if defined(ETHR_PTHREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The pthread implementation * @@ -118,7 +121,9 @@ int ethr_assert_failed(char *f, int l, char *a); #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE." #endif -#if defined(ETHR_HAVE_MIT_PTHREAD_H) +#if defined(ETHR_NEED_NPTL_PTHREAD_H) +#include <nptl/pthread.h> +#elif defined(ETHR_HAVE_MIT_PTHREAD_H) #include <pthread/mit/pthread.h> #elif defined(ETHR_HAVE_PTHREAD_H) #include <pthread.h> @@ -128,130 +133,23 @@ int ethr_assert_failed(char *f, int l, char *a); typedef pthread_t ethr_tid; -typedef struct ethr_mutex_ ethr_mutex; -struct ethr_mutex_ { - pthread_mutex_t pt_mtx; - int is_rec_mtx; - ethr_mutex *prev; - ethr_mutex *next; -#if ETHR_XCHK - int initialized; -#endif -}; - -typedef struct ethr_cond_ ethr_cond; -struct ethr_cond_ { - pthread_cond_t pt_cnd; -#if ETHR_XCHK - int initialized; -#endif -}; +typedef pthread_key_t ethr_tsd_key; -#ifndef ETHR_HAVE_PTHREAD_RWLOCK_INIT -#define ETHR_USE_RWMTX_FALLBACK -#else -typedef struct ethr_rwmutex_ ethr_rwmutex; -struct ethr_rwmutex_ { - pthread_rwlock_t pt_rwlock; -#if ETHR_XCHK - int initialized; -#endif -}; -#endif +#define ETHR_HAVE_ETHR_SIG_FUNCS 1 -/* Static initializers */ -#if ETHR_XCHK -#define ETHR_MUTEX_XCHK_INITER , ETHR_MUTEX_INITIALIZED -#define ETHR_COND_XCHK_INITER , ETHR_COND_INITIALIZED -#else -#define ETHR_MUTEX_XCHK_INITER -#define ETHR_COND_XCHK_INITER +#if defined(PURIFY) || defined(VALGRIND) +# define ETHR_FORCE_PTHREAD_RWLOCK +# define ETHR_FORCE_PTHREAD_MUTEX #endif -#define ETHR_MUTEX_INITER {PTHREAD_MUTEX_INITIALIZER, 0, NULL, NULL ETHR_MUTEX_XCHK_INITER} -#define ETHR_COND_INITER {PTHREAD_COND_INITIALIZER ETHR_COND_XCHK_INITER} - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) \ - || defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) -# define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 -# ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP -# define ETHR_REC_MUTEX_INITER \ - {PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, 1, NULL, NULL ETHR_MUTEX_XCHK_INITER} -# endif -#else -# undef ETHR_HAVE_ETHR_REC_MUTEX_INIT +#if !defined(ETHR_FORCE_PTHREAD_RWLOCK) +# define ETHR_USE_OWN_RWMTX_IMPL__ #endif -#ifndef ETHR_HAVE_PTHREAD_ATFORK -# define ETHR_NO_FORKSAFETY 1 +#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0 +# define ETHR_USE_OWN_MTX_IMPL__ #endif -typedef pthread_key_t ethr_tsd_key; - -#define ETHR_HAVE_ETHR_SIG_FUNCS 1 - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - return pthread_mutex_trylock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - return pthread_mutex_lock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - return pthread_mutex_unlock(&mtx->pt_mtx); -} - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_rdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_wrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -#endif /* ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -#endif /* ETHR_TRY_INLINE_FUNCS */ - #elif defined(ETHR_WIN32_THREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The native win32 threads implementation * @@ -273,409 +171,22 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) # undef WIN32_LEAN_AND_MEAN #endif -/* Types */ -typedef long ethr_tid; /* thread id type */ -typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; -#if ETHR_XCHK - int is_rec_mtx; -#endif -} ethr_mutex; - -typedef struct cnd_wait_event__ cnd_wait_event_; +struct ethr_join_data_; +/* Types */ typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; - cnd_wait_event_ *queue; - cnd_wait_event_ *queue_end; -} ethr_cond; - -#define ETHR_USE_RWMTX_FALLBACK - -/* Static initializers */ - -#define ETHR_MUTEX_INITER {0} -#define ETHR_COND_INITER {0} - -#define ETHR_REC_MUTEX_INITER ETHR_MUTEX_INITER - -#define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 + long id; + struct ethr_join_data_ *jdata; +} ethr_tid; /* thread id type */ typedef DWORD ethr_tsd_key; #undef ETHR_HAVE_ETHR_SIG_FUNCS -#ifdef ETHR_TRY_INLINE_FUNCS -int ethr_fake_static_mutex_init(ethr_mutex *mtx); - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return TryEnterCriticalSection(&mtx->cs) ? 0 : EBUSY; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - EnterCriticalSection(&mtx->cs); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - LeaveCriticalSection(&mtx->cs); - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#ifdef ERTS_MIXED_CYGWIN_VC - -/* atomics */ - -#ifdef _MSC_VER -# if _MSC_VER < 1300 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -# endif -# endif -# endif -# if _MSC_VER >= 1400 -# include <intrin.h> -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -#endif - -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 - -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile LONG x___ = 0; \ - (void) _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ -} while (0) - -typedef struct { - volatile LONG value; -} ethr_atomic_t; - -typedef struct { - volatile LONG locked; -} ethr_spinlock_t; - -typedef struct { - volatile LONG counter; -} ethr_rwlock_t; -#define ETHR_WLOCK_FLAG__ (((LONG) 1) << 30) - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - *i = var->value; -#else - *i = InterlockedExchangeAdd(&var->value, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - (void) InterlockedExchangeAdd(&var->value, (LONG) incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = InterlockedExchangeAdd(&var->value, (LONG) i); - *testp += i; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - (void) InterlockedIncrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - (void) InterlockedDecrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedIncrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedDecrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedAnd() provides a full - * memory barrier. - */ - *old = (long) _InterlockedAnd(&var->value, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedOr() provides a full - * memory barrier. - */ - *old = (long) _InterlockedOr(&var->value, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedCompareExchange() provides a full - * memory barrier. - */ - *old = _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) expected); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = (long) InterlockedExchange(&var->value, (LONG) new); - return 0; -} - -/* - * According to msdn InterlockedExchange() provides a full - * memory barrier. - */ - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->locked = (LONG) 0; -#else - (void) InterlockedExchange(&lock->locked, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) -{ - return 0; -} - - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - InterlockedExchange(&lock->locked, (LONG) 0); -#ifdef DEBUG - ETHR_ASSERT(old == 1); -#endif - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) -{ - LONG old; - do { - old = InterlockedExchange(&lock->locked, (LONG) 1); - } while (old != (LONG) 0); - ETHR_COMPILER_BARRIER; - return 0; -} +#define ETHR_USE_OWN_RWMTX_IMPL__ +#define ETHR_USE_OWN_MTX_IMPL__ -/* - * According to msdn InterlockedIncrement, InterlockedDecrement, - * and InterlockedExchangeAdd(), _InterlockedAnd, and _InterlockedOr - * provides full memory barriers. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->counter = (LONG) 0; -#else - (void) InterlockedExchange(&lock->counter, (LONG) 0); -#endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - InterlockedDecrement(&lock->counter); - ETHR_ASSERT(old != 0); - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - while (1) { - LONG old = InterlockedIncrement(&lock->counter); - if ((old & ETHR_WLOCK_FLAG__) == 0) - break; /* Got read lock */ - /* Restore and wait for writers to unlock */ - old = InterlockedDecrement(&lock->counter); - while (old & ETHR_WLOCK_FLAG__) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); -#endif - } - } - ETHR_COMPILER_BARRIER; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - _InterlockedAnd(&lock->counter, ~ETHR_WLOCK_FLAG__); - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - LONG old; - do { - old = _InterlockedOr(&lock->counter, ETHR_WLOCK_FLAG__); - } while (old & ETHR_WLOCK_FLAG__); - /* We got the write part of the lock; wait for readers to unlock */ - while ((old & ~ETHR_WLOCK_FLAG__) != 0) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); -#endif - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - ETHR_COMPILER_BARRIER; - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* #ifdef ERTS_MIXED_CYGWIN_VC */ +#define ETHR_YIELD() (Sleep(0), 0) #else /* No supported thread lib found */ @@ -687,6 +198,12 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) #endif +#ifdef SIZEOF_LONG +#if SIZEOF_LONG < ETHR_SIZEOF_PTR +#error size of long currently needs to be at least the same as size of void * +#endif +#endif + /* __builtin_expect() is needed by both native atomics code * and the fallback code */ #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) @@ -698,6 +215,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) # if defined(__GNUC__) # if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) # include "gcc/ethread.h" +# elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) +# include "libatomic_ops/ethread.h" # endif # ifndef ETHR_HAVE_NATIVE_ATOMICS # if ETHR_SIZEOF_PTR == 4 @@ -718,115 +237,141 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) # endif # endif # include "gcc/ethread.h" +# include "libatomic_ops/ethread.h" # endif +# elif defined(ETHR_WIN32_THREADS) +# include "win/ethread.h" # endif -#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) */ - -#ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -# undef ETHR_HAVE_NATIVE_ATOMICS -#endif -#ifdef ETHR_HAVE_OPTIMIZED_LOCKS -# undef ETHR_HAVE_NATIVE_LOCKS -#endif +#endif /* !ETHR_DISABLE_NATIVE_IMPLS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") +# endif +# ifndef ETHR_SPIN_BODY +# if defined(__i386__) || defined(__x86_64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory") +# elif defined(__ia64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory") +# elif defined(__sparc__) +# define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad") +# else +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +# endif +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +# ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0) +# endif #endif -typedef struct { - unsigned open; - ethr_mutex mtx; - ethr_cond cnd; -} ethr_gate; +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#ifndef ETHR_HAVE_NATIVE_ATOMICS /* - * Map ethread native atomics to ethread API atomics. + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when our lock based atomic fallback is used, a noop is sufficient. */ -typedef ethr_native_atomic_t ethr_atomic_t; +#define ETHR_MEMORY_BARRIER do { } while (0) +#define ETHR_WRITE_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0) #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -/* - * Map ethread native spinlocks to ethread API spinlocks. - */ -typedef ethr_native_spinlock_t ethr_spinlock_t; -/* - * Map ethread native rwlocks to ethread API rwlocks. - */ -typedef ethr_native_rwlock_t ethr_rwlock_t; +#ifndef ETHR_WRITE_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL #endif - -#ifdef ETHR_USE_RWMTX_FALLBACK -typedef struct { - ethr_mutex mtx; - ethr_cond rcnd; - ethr_cond wcnd; - unsigned readers; - unsigned waiting_readers; - unsigned waiting_writers; -#if ETHR_XCHK - int initialized; +#ifndef ETHR_READ_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER_IS_FULL #endif -} ethr_rwmutex; +#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER #endif -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -typedef long ethr_atomic_t; -#endif +#define ETHR_FATAL_ERROR__(ERR) \ + ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS +ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err); -#if defined(ETHR_WIN32_THREADS) -typedef struct { - CRITICAL_SECTION cs; -} ethr_spinlock_t; -typedef struct { - CRITICAL_SECTION cs; - unsigned counter; -} ethr_rwlock_t; +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif -int ethr_do_spinlock_init(ethr_spinlock_t *lock); -int ethr_do_rwlock_init(ethr_rwlock_t *lock); +#ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +#endif -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) +#ifndef ETHR_YIELD +# if defined(ETHR_HAVE_SCHED_YIELD) +# ifdef ETHR_HAVE_SCHED_H +# include <sched.h> +# endif +# include <errno.h> +# if defined(ETHR_SCHED_YIELD_RET_INT) +# define ETHR_YIELD() (sched_yield() < 0 ? errno : 0) +# else +# define ETHR_YIELD() (sched_yield(), 0) +# endif +# elif defined(ETHR_HAVE_PTHREAD_YIELD) +# if defined(ETHR_PTHREAD_YIELD_RET_INT) +# define ETHR_YIELD() pthread_yield() +# else +# define ETHR_YIELD() (pthread_yield(), 0) +# endif +# else +# define ETHR_YIELD() (ethr_compiler_barrier(), 0) +# endif +#endif + +#include "ethr_optimized_fallbacks.h" -#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -typedef struct { - pthread_spinlock_t spnlck; -} ethr_spinlock_t; typedef struct { - pthread_spinlock_t spnlck; - unsigned counter; -} ethr_rwlock_t; -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) + void *(*thread_create_prepare_func)(void); + void (*thread_create_parent_func)(void *); + void (*thread_create_child_func)(void *); +} ethr_init_data; -#else /* ethr mutex/rwmutex */ +#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL} typedef struct { - ethr_mutex mtx; -} ethr_spinlock_t; + void *(*alloc)(size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); +} ethr_memory_allocator; + +#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL} typedef struct { - ethr_rwmutex rwmtx; -} ethr_rwlock_t; + ethr_memory_allocator std; + ethr_memory_allocator sl; + ethr_memory_allocator ll; +} ethr_memory_allocators; -#endif /* end mutex/rwmutex */ -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ +#define ETHR_MEM_ALLOCS_DEF_INITER__ \ + {ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__} typedef struct { - void *(*alloc)(size_t); - void *(*realloc)(void *, size_t); - void (*free)(void *); - void *(*thread_create_prepare_func)(void); - void (*thread_create_parent_func)(void *); - void (*thread_create_child_func)(void *); -} ethr_init_data; + ethr_memory_allocators mem; + int reader_groups; + int main_threads; +} ethr_late_init_data; -#define ETHR_INIT_DATA_DEFAULT_INITER {malloc, realloc, free, NULL, NULL, NULL} +#define ETHR_LATE_INIT_DATA_DEFAULT_INITER \ + {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0} typedef struct { int detached; /* boolean (default false) */ @@ -835,18 +380,15 @@ typedef struct { #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1} -#if defined(ETHR_CUSTOM_INLINE_FUNC_NAME_) || !defined(ETHR_TRY_INLINE_FUNCS) -# define ETHR_NEED_MTX_PROTOTYPES__ -# define ETHR_NEED_RWMTX_PROTOTYPES__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) # define ETHR_NEED_SPINLOCK_PROTOTYPES__ +# define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ # define ETHR_NEED_ATOMIC_PROTOTYPES__ #endif -#if !defined(ETHR_NEED_RWMTX_PROTOTYPES__) && defined(ETHR_USE_RWMTX_FALLBACK) -# define ETHR_NEED_RWMTX_PROTOTYPES__ -#endif - int ethr_init(ethr_init_data *); +int ethr_late_init(ethr_late_init_data *); int ethr_install_exit_handler(void (*funcp)(void)); int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *); int ethr_thr_join(ethr_tid, void **); @@ -854,65 +396,6 @@ int ethr_thr_detach(ethr_tid); void ethr_thr_exit(void *); ethr_tid ethr_self(void); int ethr_equal_tids(ethr_tid, ethr_tid); -int ethr_mutex_init(ethr_mutex *); -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT -int ethr_rec_mutex_init(ethr_mutex *); -#endif -int ethr_mutex_destroy(ethr_mutex *); -int ethr_mutex_set_forksafe(ethr_mutex *); -int ethr_mutex_unset_forksafe(ethr_mutex *); -#ifdef ETHR_NEED_MTX_PROTOTYPES__ -int ethr_mutex_trylock(ethr_mutex *); -int ethr_mutex_lock(ethr_mutex *); -int ethr_mutex_unlock(ethr_mutex *); -#endif -int ethr_cond_init(ethr_cond *); -int ethr_cond_destroy(ethr_cond *); -int ethr_cond_signal(ethr_cond *); -int ethr_cond_broadcast(ethr_cond *); -int ethr_cond_wait(ethr_cond *, ethr_mutex *); -int ethr_cond_timedwait(ethr_cond *, ethr_mutex *, ethr_timeval *); - -int ethr_rwmutex_init(ethr_rwmutex *); -int ethr_rwmutex_destroy(ethr_rwmutex *); -#ifdef ETHR_NEED_RWMTX_PROTOTYPES__ -int ethr_rwmutex_tryrlock(ethr_rwmutex *); -int ethr_rwmutex_rlock(ethr_rwmutex *); -int ethr_rwmutex_runlock(ethr_rwmutex *); -int ethr_rwmutex_tryrwlock(ethr_rwmutex *); -int ethr_rwmutex_rwlock(ethr_rwmutex *); -int ethr_rwmutex_rwunlock(ethr_rwmutex *); -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -int ethr_atomic_init(ethr_atomic_t *, long); -int ethr_atomic_set(ethr_atomic_t *, long); -int ethr_atomic_read(ethr_atomic_t *, long *); -int ethr_atomic_inctest(ethr_atomic_t *, long *); -int ethr_atomic_dectest(ethr_atomic_t *, long *); -int ethr_atomic_inc(ethr_atomic_t *); -int ethr_atomic_dec(ethr_atomic_t *); -int ethr_atomic_addtest(ethr_atomic_t *, long, long *); -int ethr_atomic_add(ethr_atomic_t *, long); -int ethr_atomic_and_old(ethr_atomic_t *, long, long *); -int ethr_atomic_or_old(ethr_atomic_t *, long, long *); -int ethr_atomic_xchg(ethr_atomic_t *, long, long *); -int ethr_atomic_cmpxchg(ethr_atomic_t *, long, long, long *); -#endif - -#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ -int ethr_spinlock_init(ethr_spinlock_t *); -int ethr_spinlock_destroy(ethr_spinlock_t *); -int ethr_spin_unlock(ethr_spinlock_t *); -int ethr_spin_lock(ethr_spinlock_t *); - -int ethr_rwlock_init(ethr_rwlock_t *); -int ethr_rwlock_destroy(ethr_rwlock_t *); -int ethr_read_unlock(ethr_rwlock_t *); -int ethr_read_lock(ethr_rwlock_t *); -int ethr_write_unlock(ethr_rwlock_t *); -int ethr_write_lock(ethr_rwlock_t *); -#endif int ethr_time_now(ethr_timeval *); int ethr_tsd_key_create(ethr_tsd_key *); @@ -920,13 +403,6 @@ int ethr_tsd_key_delete(ethr_tsd_key); int ethr_tsd_set(ethr_tsd_key, void *); void *ethr_tsd_get(ethr_tsd_key); -int ethr_gate_init(ethr_gate *); -int ethr_gate_destroy(ethr_gate *); -int ethr_gate_close(ethr_gate *); -int ethr_gate_let_through(ethr_gate *, unsigned); -int ethr_gate_wait(ethr_gate *); -int ethr_gate_swait(ethr_gate *, int); - #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #include <signal.h> int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset); @@ -935,534 +411,579 @@ int ethr_sigwait(const sigset_t *set, int *sig); void ethr_compiler_barrier(void); -#ifdef ETHR_TRY_INLINE_FUNCS - -#ifdef ETHR_HAVE_NATIVE_ATOMICS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_init(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_set(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ - *i = ethr_native_atomic_read(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ethr_native_atomic_add(var, incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = ethr_native_atomic_add_return(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - ethr_native_atomic_inc(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - ethr_native_atomic_dec(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_inc_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_dec_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_and_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_or_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = ethr_native_atomic_xchg(var, new); - return 0; -} - -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_cmpxchg(var, new, expected); - return 0; -} +#if defined(ETHR_HAVE_NATIVE_SPINLOCKS) +typedef ethr_native_spinlock_t ethr_spinlock_t; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) +typedef ethr_opt_spinlock_t ethr_spinlock_t; +#elif defined(__WIN32__) +typedef CRITICAL_SECTION ethr_spinlock_t; +#else +typedef pthread_mutex_t ethr_spinlock_t; +#endif -#endif /* ETHR_HAVE_NATIVE_ATOMICS */ +#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ +int ethr_spinlock_init(ethr_spinlock_t *); +int ethr_spinlock_destroy(ethr_spinlock_t *); +void ethr_spin_unlock(ethr_spinlock_t *); +void ethr_spin_lock(ethr_spinlock_t *); +#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spinlock_init(lock); return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX)) + return ethr_win_get_errno__(); + return 0; +#else + return pthread_mutex_init((pthread_mutex_t *) lock, NULL); +#endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + DeleteCriticalSection((CRITICAL_SECTION *) lock); + return 0; +#else + return pthread_mutex_destroy((pthread_mutex_t *) lock); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_unlock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + LeaveCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_unlock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ - ethr_native_rwlock_init(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_read_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - ethr_native_read_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_write_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - ethr_native_write_lock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + EnterCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_lock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -#endif /* ETHR_HAVE_NATIVE_LOCKS */ - #endif /* ETHR_TRY_INLINE_FUNCS */ +#ifdef ETHR_HAVE_NATIVE_ATOMICS /* - * Fallbacks for atomics used in absence of optimized implementation. + * Map ethread native atomics to ethread API atomics. */ -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS +typedef ethr_native_atomic_t ethr_atomic_t; +#else +typedef long ethr_atomic_t; +#endif +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +void ethr_atomic_init(ethr_atomic_t *, long); +void ethr_atomic_set(ethr_atomic_t *, long); +long ethr_atomic_read(ethr_atomic_t *); +long ethr_atomic_inc_read(ethr_atomic_t *); +long ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +long ethr_atomic_add_read(ethr_atomic_t *, long); +void ethr_atomic_add(ethr_atomic_t *, long); +long ethr_atomic_read_band(ethr_atomic_t *, long); +long ethr_atomic_read_bor(ethr_atomic_t *, long); +long ethr_atomic_xchg(ethr_atomic_t *, long); +long ethr_atomic_cmpxchg(ethr_atomic_t *, long, long); +long ethr_atomic_read_acqb(ethr_atomic_t *); +long ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, long); +void ethr_atomic_dec_relb(ethr_atomic_t *); +long ethr_atomic_dec_read_relb(ethr_atomic_t *); +long ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, long, long); +long ethr_atomic_cmpxchg_relb(ethr_atomic_t *, long, long); +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS /* - * ETHR_MEMORY_BARRIER orders between locked and atomic accesses only, - * i.e. when this atomic fallback is used a noop is sufficient. + * Fallbacks for atomics used in absence of a native implementation. */ -#define ETHR_MEMORY_BARRIER #define ETHR_ATOMIC_ADDR_BITS 10 #define ETHR_ATOMIC_ADDR_SHIFT 6 typedef struct { union { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - pthread_spinlock_t spnlck; -#else - ethr_mutex mtx; -#endif + ethr_spinlock_t lck; char buf[ETHR_CACHE_LINE_SIZE]; } u; } ethr_atomic_protection_t; extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - #define ETHR_ATOMIC_PTR2LCK__(PTR) \ (ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.spnlck) + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) #define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ do { \ - pthread_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = pthread_spin_lock(slp__); \ - if (res__ != 0) \ - return res__; \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ { EXPS; } \ - return pthread_spin_unlock(slp__); \ + ethr_spin_unlock(slp__); \ } while (0) -#else /* ethread mutex */ - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.mtx) - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_mutex *mtxp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(mtxp__); \ - if (res__ != 0) \ - return res__; \ - { EXPS; } \ - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(mtxp__); \ -} while (0) - -#endif /* end ethread mutex */ - -#ifdef ETHR_TRY_INLINE_FUNCS +#endif -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_init(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *i = (long) *var); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) *var); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *incp, long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *testp = (long) ++(*incp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_add(var, incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, long i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_add_return(var, i); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *decp, long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, *testp = (long) --(*decp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_inc(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *incp, - long i, - long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *incp += i; *testp = *incp); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *incp) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, ++(*incp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) ++(*var)); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *decp) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, --(*decp)); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return(var); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) --(*var)); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + long mask) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var &= mask); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_and_retold(var, mask); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + long mask) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var |= mask); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_or_retold(var, mask); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif } -static ETHR_INLINE int +static ETHR_INLINE long ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) + long new) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var = new); -} +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_xchg(var, new); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int +static ETHR_INLINE long ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, long new, - long expected, - long *old) + long exp) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__( - var, - long old_val = *var; - *old = old_val; - if (__builtin_expect(old_val == expected, 1)) - *var = new; - ); - return 0; +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg(var, new, exp); +#else + long res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - /* - * Fallbacks for spin locks, and rw spin locks used in absence of - * optimized implementation. + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier */ -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS -#ifdef ETHR_TRY_INLINE_FUNCS +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_init(&lock->spnlck, 0); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return_acqb(var); #else - return ethr_mutex_init(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); #endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, long val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set_relb(var, val); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec_relb(var); #else - return ethr_mutex_destroy(&lock->mtx); + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); #endif } +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return_relb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + long new, + long exp) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_acqb(var, new, exp); #else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); #endif } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) +static ETHR_INLINE long +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + long new, + long exp) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_lock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_relb(var, new, exp); #else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(&lock->mtx); + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); #endif } -#ifdef ETHR_USE_RWMTX_FALLBACK -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) X +#endif /* ETHR_TRY_INLINE_FUNCS */ + +typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ + +#if defined(ETHR_WIN32_THREADS) +# include "win/ethr_event.h" #else -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) ETHR_INLINE_FUNC_NAME_(X) +# include "pthread/ethr_event.h" +#endif + +int ethr_set_main_thr_status(int, int); +int ethr_get_main_thr_status(int *); + +struct ethr_ts_event_ { + ethr_ts_event *next; + ethr_ts_event *prev; + ethr_event event; + void *udata; + ethr_atomic_t uaflgs; + unsigned uflgs; + unsigned iflgs; /* for ethr lib only */ + short rgix; /* for ethr lib only */ + short mtix; /* for ethr lib only */ +}; + +#define ETHR_TS_EV_ETHREAD (((unsigned) 1) << 0) +#define ETHR_TS_EV_INITED (((unsigned) 1) << 1) +#define ETHR_TS_EV_TMP (((unsigned) 1) << 2) +#define ETHR_TS_EV_MAIN_THR (((unsigned) 1) << 3) + +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp); +int ethr_free_ts_event__(ethr_ts_event *tsep); +int ethr_make_ts_event__(ethr_ts_event **tsepp); + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) +ethr_ts_event *ethr_get_ts_event(void); +void ethr_leave_ts_event(ethr_ts_event *); +#endif + +#if defined(ETHR_PTHREADS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) + +extern pthread_key_t ethr_ts_event_key__; + +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) +{ + ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_make_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) +{ + +} + #endif +#elif defined(ETHR_WIN32_THREADS) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) + +extern DWORD ethr_ts_event_key__; + +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) +{ + ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_get_tmp_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) +{ + if (tsep->iflgs & ETHR_TS_EV_TMP) { + int res = ethr_free_ts_event__(tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +#endif + +#endif + +#include "ethr_mutex.h" /* Need atomic declarations and tse */ + +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +typedef ethr_native_rwlock_t ethr_rwlock_t; +#else +typedef ethr_rwmutex ethr_rwlock_t; +#endif + +#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__ +int ethr_rwlock_init(ethr_rwlock_t *); +int ethr_rwlock_destroy(ethr_rwlock_t *); +void ethr_read_unlock(ethr_rwlock_t *); +void ethr_read_lock(ethr_rwlock_t *); +void ethr_write_unlock(ethr_rwlock_t *); +void ethr_write_lock(ethr_rwlock_t *); +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_init(&lock->spnlck, 0); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_rwlock_init(lock); + return 0; #else - return ethr_rwmutex_init(&lock->rwmtx); + return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL); #endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + return 0; #else - return ethr_rwmutex_destroy(&lock->rwmtx); + return ethr_rwmutex_destroy((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter--; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_runlock)(&lock->rwmtx); + ethr_rwmutex_runlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int locked = 0; - do { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - if ((lock->counter & ETHR_RWLOCK_WRITERS) == 0) { - lock->counter++; - locked = 1; - } - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } while (!locked); - return 0; +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rlock)(&lock->rwmtx); + ethr_rwmutex_rlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwunlock)(&lock->rwmtx); + ethr_rwmutex_rwunlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - while (1) { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter |= ETHR_RWLOCK_WRITERS; - if (lock->counter == ETHR_RWLOCK_WRITERS) - return 0; - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwlock)(&lock->rwmtx); + ethr_rwmutex_rwlock((ethr_rwmutex *) lock); #endif } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ - -#if defined(ETHR_HAVE_OPTIMIZED_LOCKS) || defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -# define ETHR_HAVE_OPTIMIZED_SPINLOCK -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* #ifndef ETHREAD_H__ */ diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index c9fd87c2f6..5debb44756 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -29,26 +29,54 @@ /* Define if you have pthreads */ #undef ETHR_PTHREADS +/* Define if you need the <nptl/pthread.h> header file. */ +#undef ETHR_NEED_NPTL_PTHREAD_H + /* Define if you have the <pthread.h> header file. */ #undef ETHR_HAVE_PTHREAD_H /* Define if the pthread.h header file is in pthread/mit directory. */ #undef ETHR_HAVE_MIT_PTHREAD_H -/* Define if you have the pthread_mutexattr_settype function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE +/* Define if you have the pthread_spin_lock function. */ +#undef ETHR_HAVE_PTHREAD_SPIN_LOCK -/* Define if you have the pthread_mutexattr_setkind_np function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP +/* Define if you want to force usage of pthread rwlocks */ +#undef ETHR_FORCE_PTHREAD_RWLOCK -/* Define if you have the pthread_atfork function. */ -#undef ETHR_HAVE_PTHREAD_ATFORK +/* Define if you have the pthread_rwlockattr_setkind_np() function. */ +#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP -/* Define if you have the pthread_spin_lock function. */ -#undef ETHR_HAVE_PTHREAD_SPIN_LOCK +/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock + attribute. */ +#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP -/* Define if you have a pthread_rwlock implementation that can be used */ -#undef ETHR_HAVE_PTHREAD_RWLOCK_INIT +/* Define if you have a linux futex implementation. */ +#undef ETHR_HAVE_LINUX_FUTEX + +/* Define if you have gcc atomic operations */ +#undef ETHR_HAVE_GCC_ATOMIC_OPS + +/* Define if you prefer gcc native ethread implementations */ +#undef ETHR_PREFER_GCC_NATIVE_IMPLS + +/* Define if you have the <sched.h> header file. */ +#undef ETHR_HAVE_SCHED_H + +/* Define if you have the sched_yield() function. */ +#undef ETHR_HAVE_SCHED_YIELD + +/* Define if you have the pthread_yield() function. */ +#undef ETHR_HAVE_PTHREAD_YIELD + +/* Define if pthread_yield() returns an int. */ +#undef ETHR_PTHREAD_YIELD_RET_INT + +/* Define if sched_yield() returns an int. */ +#undef ETHR_SCHED_YIELD_RET_INT + +/* Define if you want compatibilty with x86 processors before pentium4. */ +#undef ETHR_PRE_PENTIUM4_COMPAT /* Define if you have the pthread_rwlockattr_setkind_np() function. */ #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP @@ -63,6 +91,15 @@ /* Define if you prefer gcc native ethread implementations */ #undef ETHR_PREFER_GCC_NATIVE_IMPLS +/* Define if you have libatomic_ops atomic operations */ +#undef ETHR_HAVE_LIBATOMIC_OPS + +/* Define if you prefer libatomic_ops native ethread implementations */ +#undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS + +/* Define to the size of AO_t if libatomic_ops is used */ +#undef ETHR_SIZEOF_AO_T + /* Define if you want to turn on extra sanity checking in the ethread library */ #undef ETHR_XCHK diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 775030c8d5..5fe6e23477 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -55,6 +55,7 @@ do { \ volatile long x___ = 0; \ (void) __sync_val_compare_and_swap(&x___, (long) 0, (long) 1); \ } while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) @@ -157,6 +158,24 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) return act; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + return __sync_add_and_fetch(&var->counter, (long) 0); +} + +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return + +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif #endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 90b4c5f773..f28258059f 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -32,8 +32,11 @@ typedef struct { volatile long counter; } ethr_native_atomic_t; -#ifdef __x86_64__ +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) #define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") +#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") +#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") #else #define ETHR_MEMORY_BARRIER \ do { \ @@ -42,7 +45,9 @@ do { \ } while (0) #endif -#ifdef ETHR_TRY_INLINE_FUNCS +#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #ifdef __x86_64__ #define LONG_SUFFIX "q" @@ -158,6 +163,22 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) return tmp; } +/* + * Atomic ops with at least specified barriers. + */ + +#define ethr_native_atomic_read_acqb ethr_native_atomic_read +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) +#define ethr_native_atomic_set_relb ethr_native_atomic_set +#else +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#endif +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #undef LONG_SUFFIX #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index fad8b108fa..ed43e77279 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_I386_ETHREAD_H */ diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h index c009be8ef1..be47f459ce 100644 --- a/erts/include/internal/i386/rwlock.h +++ b/erts/include/internal/i386/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #define ETHR_RWLOCK_OFFSET (1<<24) diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h index 2b4832e26a..0325324895 100644 --- a/erts/include/internal/i386/spinlock.h +++ b/erts/include/internal/i386/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) @@ -46,7 +46,7 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock) * On i386 this needs to be a locked operation * to avoid Pentium Pro errata 66 and 92. */ -#if defined(__x86_64__) +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) __asm__ __volatile__("" : : : "memory"); *(unsigned char*)&lock->lock = 0; #else diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h new file mode 100644 index 0000000000..a6eb43a0bd --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -0,0 +1,292 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__ +#define ETHR_LIBATOMIC_OPS_ATOMIC_H__ + +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \ + || defined(__x86_64__) +#define AO_USE_PENTIUM4_INSTRS +#endif + +#include "atomic_ops.h" + +/* + * libatomic_ops can be downloaded from: + * http://www.hpl.hp.com/research/linux/atomic_ops/ + * + * These operations need to be defined by libatomic_ops; + * otherwise, we won't compile: + * - AO_nop_full() + * - AO_load() + * - AO_store() + * - AO_compare_and_swap() + * + * The `AO_t' type also have to be at least as large as + * `void *' and `long' types. + */ + +#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR +#error The AO_t type is too small +#endif + +typedef struct { + volatile AO_t counter; +} ethr_native_atomic_t; + +#define ETHR_MEMORY_BARRIER AO_nop_full() +#ifdef AO_HAVE_nop_write +# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write() +#else +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_HAVE_nop_read +# define ETHR_READ_MEMORY_BARRIER AO_nop_read() +#else +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_NO_DD_ORDERING +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory") +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_atomic_set(ethr_native_atomic_t *var, long value) +{ + AO_store(&var->counter, (AO_t) value); +} + +static ETHR_INLINE void +ethr_native_atomic_init(ethr_native_atomic_t *var, long value) +{ + ethr_native_atomic_set(var, value); +} + +static ETHR_INLINE long +ethr_native_atomic_read(ethr_native_atomic_t *var) +{ + return (long) AO_load(&var->counter); +} + +static ETHR_INLINE long +ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +{ +#ifdef AO_HAVE_fetch_and_add + return ((long) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; +#else + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp + (AO_t) incr; + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) new; + } +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +{ + (void) ethr_native_atomic_add_return(var, incr); +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_add1 + return ((long) AO_fetch_and_add1(&var->counter)) + 1; +#else + return ethr_native_atomic_add_return(var, 1); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_inc(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_inc_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_sub1 + return ((long) AO_fetch_and_sub1(&var->counter)) - 1; +#else + return ethr_native_atomic_add_return(var, -1); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_dec(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_dec_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp & ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) exp; + } +} + +static ETHR_INLINE long +ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp | ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (long) exp; + } +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long exp) +{ + long act; + do { + if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + return act; +} + +static ETHR_INLINE long +ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) + return (long) exp; + } +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_load_acquire + return (long) AO_load_acquire(&var->counter); +#else + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_add1_acquire + return ((long) AO_fetch_and_add1_acquire(&var->counter)) + 1; +#else + long res = ethr_native_atomic_add_return(var, 1); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long value) +{ +#ifdef AO_HAVE_store_release + AO_store_release(&var->counter, (AO_t) value); +#else + ETHR_MEMORY_BARRIER; + ethr_native_atomic_set(var, value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ +#ifdef AO_HAVE_fetch_and_sub1_release + return ((long) AO_fetch_and_sub1_release(&var->counter)) - 1; +#else + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_dec_return(var); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + (void) ethr_native_atomic_dec_return_relb(var); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +{ +#ifdef AO_HAVE_compare_and_swap_acquire + long act; + do { + if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + AO_nop_full(); + return act; +#else + long act = ethr_native_atomic_cmpxchg(var, new, exp); + ETHR_MEMORY_BARRIER; + return act; +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +{ +#ifdef AO_HAVE_compare_and_swap_release + long act; + do { + if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (long) AO_load(&var->counter); + } while (act == exp); + return act; +#else + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_cmpxchg(var, new, exp); +#endif +} + + +#endif + +#endif + +#endif diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h new file mode 100644 index 0000000000..ee73ba73bc --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHREAD_LIBATOMIC_OPS_H__ +#define ETHREAD_LIBATOMIC_OPS_H__ + +#include "ethr_atomic.h" + +#endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index 105d874995..f21f7c9588 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -34,7 +34,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, int i) @@ -205,6 +205,26 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) return old; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +#define ethr_native_atomic_set_relb ethr_native_atomic_xchg +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return +#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return + +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_PPC_ATOMIC_H */ diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index d2a72c3dc1..12efc1b653 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_PPC32_ETHREAD_H */ diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h index 9bdab12826..19ec26ab68 100644 --- a/erts/include/internal/ppc32/rwlock.h +++ b/erts/include/internal/ppc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h index 034c20c143..c8460a3e8a 100644 --- a/erts/include/internal/ppc32/spinlock.h +++ b/erts/include/internal/ppc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h new file mode 100644 index 0000000000..104ec287e0 --- /dev/null +++ b/erts/include/internal/pthread/ethr_event.h @@ -0,0 +1,151 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Linux futex implementation of ethread events ------------------------- */ +#define ETHR_LINUX_FUTEX_IMPL__ + +#include <sys/syscall.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/time.h> + +/* + * Note: Linux futexes operate on 32-bit integers, but + * ethr_native_atomic_t are 64-bits on 64-bit + * platforms. This has to be taken into account. + * Therefore, in each individual value used each + * byte look the same. + */ + +#if ETHR_SIZEOF_PTR == 8 + +#define ETHR_EVENT_OFF_WAITER__ 0xffffffffffffffffL +#define ETHR_EVENT_OFF__ 0x7777777777777777L +#define ETHR_EVENT_ON__ 0L + +#elif ETHR_SIZEOF_PTR == 4 + +#define ETHR_EVENT_OFF_WAITER__ 0xffffffffL +#define ETHR_EVENT_OFF__ 0x77777777L +#define ETHR_EVENT_ON__ 0L + +#else + +#error ehrm... + +#endif + +#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE) +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE +#else +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE +#endif + +typedef struct { + ethr_atomic_t futex; +} ethr_event; + +#define ETHR_FUTEX__(FTX, OP, VAL) \ + (-1 == syscall(__NR_futex, (void *) (FTX), (OP), (int) (VAL), NULL, NULL, 0)\ + ? errno : 0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + long val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic_xchg(&e->futex, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic_set(&e->futex, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +typedef struct { + ethr_atomic_t state; + pthread_mutex_t mtx; + pthread_cond_t cnd; +} ethr_event; + +#define ETHR_EVENT_OFF_WAITER__ -1L +#define ETHR_EVENT_OFF__ 1L +#define ETHR_EVENT_ON__ 0L + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + long val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic_xchg(&e->state, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_cond_signal(&e->cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_mutex_unlock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 8fde449a52..2a995d4465 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -32,7 +32,7 @@ typedef struct { __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ : : : "memory") -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #if defined(__arch64__) #define CASX "casx" @@ -172,6 +172,43 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) return new; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + return res; +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ethr_native_atomic_set(var, i); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + ethr_native_atomic_dec(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore"); + return ethr_native_atomic_dec_return(var); +} + +#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg +#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHR_SPARC32_ATOMIC_H */ diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index 1d55399640..dca113b4d6 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_SPARC32_ETHREAD_H */ diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h index 12448e0b06..465ec96866 100644 --- a/erts/include/internal/sparc32/rwlock.h +++ b/erts/include/internal/sparc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h index b4fe48b714..493d514210 100644 --- a/erts/include/internal/sparc32/spinlock.h +++ b/erts/include/internal/sparc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile unsigned char lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 5e4c7ac9fe..69569d82d1 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -34,7 +34,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __insn_mf() -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, long i) @@ -45,7 +45,6 @@ ethr_native_atomic_init(ethr_native_atomic_t *var, long i) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, long i) { - __insn_mf(); atomic_exchange_acq(&var->counter, i); } @@ -58,28 +57,24 @@ ethr_native_atomic_read(ethr_native_atomic_t *var) static ETHR_INLINE void ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) { - __insn_mf(); atomic_add(&var->counter, incr); } static ETHR_INLINE void ethr_native_atomic_inc(ethr_native_atomic_t *var) { - __insn_mf(); atomic_increment(&var->counter); } static ETHR_INLINE void ethr_native_atomic_dec(ethr_native_atomic_t *var) { - __insn_mf(); atomic_decrement(&var->counter); } static ETHR_INLINE long ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) { - __insn_mf(); return atomic_exchange_and_add(&var->counter, incr) + incr; } @@ -98,33 +93,81 @@ ethr_native_atomic_dec_return(ethr_native_atomic_t *var) static ETHR_INLINE long ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_and_val(&var->counter, mask); } static ETHR_INLINE long ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) { - __insn_mf(); return atomic_or_val(&var->counter, mask); } static ETHR_INLINE long ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) { - __insn_mf(); return atomic_exchange_acq(&var->counter, val); } static ETHR_INLINE long ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ + long res = ethr_native_atomic_inc_return(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long val) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic_set(var, val); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic_dec(var); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_dec_return(var); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +{ + return ethr_native_atomic_cmpxchg(var, new, exp); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic_cmpxchg(var, new, exp); +} + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_TILE_ATOMIC_H */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h new file mode 100644 index 0000000000..500459dd6c --- /dev/null +++ b/erts/include/internal/win/ethr_atomic.h @@ -0,0 +1,244 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support when using VC++ + * Author: Rickard Green + */ + +#ifndef ETHR_WIN_ATOMIC_H__ +#define ETHR_WIN_ATOMIC_H__ + +#ifdef _MSC_VER +# if _MSC_VER < 1300 +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ +# else +# if defined(_M_IX86) +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +# else /* I.e. IA64 */ +# if _MSC_VER >= 1400 +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +# else +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +# endif +# endif +# endif +# if _MSC_VER >= 1400 +# include <intrin.h> +# undef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +#pragma intrinsic(_ReadWriteBarrier) +#pragma intrinsic(_InterlockedAnd) +#pragma intrinsic(_InterlockedOr) +#else +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +#endif + +/* + * No configure test checking for _Interlocked*_{acq,rel} and + * Interlocked*{Acquire,Release} have been written yet... + * + * Note, that these are pure optimizations for the itanium + * processor. + */ + +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ +#pragma intrinsic(_InterlockedCompareExchange_acq) +#endif +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL +#pragma intrinsic(_InterlockedCompareExchange_rel) +#endif + + +typedef struct { + volatile LONG value; +} ethr_native_atomic_t; + +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile LONG x___ = 0; \ + _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ +} while (0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +{ + var->value = (LONG) i; +} + +static ETHR_INLINE void +ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + var->value = (LONG) i; +#else + (void) InterlockedExchange(&var->value, (LONG) i); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_read(ethr_native_atomic_t *var) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + return var->value; +#else + return InterlockedExchangeAdd(&var->value, (LONG) 0); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +{ + (void) InterlockedExchangeAdd(&var->value, (LONG) incr); +} + +static ETHR_INLINE long +ethr_native_atomic_add_return(ethr_native_atomic_t *var, long i) +{ + LONG tmp = InterlockedExchangeAdd(&var->value, (LONG) i); + return tmp + i; +} + +static ETHR_INLINE void +ethr_native_atomic_inc(ethr_native_atomic_t *var) +{ + (void) InterlockedIncrement(&var->value); +} + +static ETHR_INLINE void +ethr_native_atomic_dec(ethr_native_atomic_t *var) +{ + (void) InterlockedDecrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +{ + return (long) InterlockedIncrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +{ + return (long) InterlockedDecrement(&var->value); +} + +static ETHR_INLINE long +ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +{ + return (long) _InterlockedAnd(&var->value, mask); +} + +static ETHR_INLINE long +ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +{ + return (long) _InterlockedOr(&var->value, mask); +} + + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +{ + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +} + + +static ETHR_INLINE long +ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +{ + return (long) InterlockedExchange(&var->value, (LONG) new); +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDEXCHANGEADDACQUIRE + return (long) InterlockedExchangeAddAcquire(&var->value, (LONG) 0); +#else + return (long) InterlockedExchangeAdd(&var->value, (LONG) 0); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDINCREMENTACQUIRE + return (long) InterlockedIncrementAcquire(&var->value); +#else + return (long) InterlockedIncrement(&var->value); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +{ + (void) InterlockedExchange(&var->value, (LONG) i); +} + +static ETHR_INLINE void +ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE + (void) InterlockedDecrementRelease(&var->value); +#else + (void) InterlockedDecrement(&var->value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +{ +#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE + return (long) InterlockedDecrementRelease(&var->value); +#else + return (long) InterlockedDecrement(&var->value); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long old) +{ +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ + return (long) _InterlockedCompareExchange_acq(&var->value, (LONG) new, (LONG) old); +#else + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +#endif +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long old) +{ + +#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL + return (long) _InterlockedCompareExchange_rel(&var->value, (LONG) new, (LONG) old); +#else + return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); +#endif +} + +#endif + +#endif diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h new file mode 100644 index 0000000000..af57c20f91 --- /dev/null +++ b/erts/include/internal/win/ethr_event.h @@ -0,0 +1,62 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_EVENT_OFF_WAITER__ ((LONG) -1) +#define ETHR_EVENT_OFF__ ((LONG) 1) +#define ETHR_EVENT_ON__ ((LONG) 0) + +typedef struct { + volatile LONG state; + HANDLE handle; +} ethr_event; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + /* InterlockedExchange() imply a full memory barrier which is important */ + LONG state = InterlockedExchange(&e->state, ETHR_EVENT_ON__); + if (state == ETHR_EVENT_OFF_WAITER__) { + if (!SetEvent(e->handle)) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + /* InterlockedExchange() imply a full memory barrier which is important */ + InterlockedExchange(&e->state, ETHR_EVENT_OFF__); +} + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h new file mode 100644 index 0000000000..b52710f6a3 --- /dev/null +++ b/erts/include/internal/win/ethread.h @@ -0,0 +1,31 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomic and spinlock ethread support when using VC++ + * Author: Rickard Green + */ + +#ifndef ETHREAD_WIN_H__ +#define ETHREAD_WIN_H__ + +#include "ethr_atomic.h" +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index e7caac8072..0d3181cace 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -280,8 +280,13 @@ endif # # ethread library # +ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@ ifneq ($(strip $(ETHR_LIB_NAME)),) -ETHREAD_LIB_SRC=common/ethread.c +ETHREAD_LIB_SRC=common/ethr_aux.c \ + common/ethr_mutex.c \ + common/ethr_cbf.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethr_event.c ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX) ifeq ($(USING_VC),yes) @@ -379,13 +384,13 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(r_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(r_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ $(OBJ_DIR)/%.o: common/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ # Win32 specific @@ -393,25 +398,25 @@ $(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c $(MD_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ -$(MD_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MD_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ $(MDd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ -$(MDd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MDd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ $(MT_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ -$(MT_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MT_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ $(MTd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ -$(MTd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MTd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ # @@ -438,6 +443,8 @@ RELEASE_LIBS=$(ERTS_LIBS) INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/README \ $(ERTS_INCL_INT)/ethread.h \ + $(ERTS_INCL_INT)/ethr_mutex.h \ + $(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \ $(ERTS_INCL_INT)/$(TARGET)/ethread.mk \ $(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \ $(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \ @@ -447,7 +454,8 @@ INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/erl_misc_utils.h \ $(ERTS_INCL_INT)/erl_errno.h -INTERNAL_X_RELEASE_INCLUDE_DIRS= i386 x86_64 ppc32 sparc32 sparc64 tile gcc +INTERNAL_X_RELEASE_INCLUDE_DIRS= \ + i386 x86_64 ppc32 sparc32 sparc64 tile gcc pthread win libatomic_ops INTERNAL_RELEASE_LIBS= \ ../lib/internal/README \ diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index f70db86960..d2ef7140a5 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -59,8 +59,25 @@ # endif #endif -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(HAVE_SCHED_xETAFFINITY) # include <sched.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP) \ + (sched_getaffinity((CPUINFOP)->pid, \ + sizeof(cpu_set_t), \ + &(CPUINFOP)->cpuset) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(SETP) \ + (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0) +#elif defined(__WIN32__) +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +# define cpu_set_t DWORD +# define CPU_SETSIZE (sizeof(DWORD)*8) +# define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0) +# define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU))) +# define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU))) +# define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0) +#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity +#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity #endif #ifdef HAVE_PSET_INFO # include <sys/pset.h> @@ -105,25 +122,58 @@ struct erts_cpu_info_t_ { int available; int topology_size; erts_cpu_topology_t *topology; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *affinity_str; char affinity_str_buf[CPU_SETSIZE/4+2]; cpu_set_t cpuset; +#if defined(HAVE_SCHED_xETAFFINITY) pid_t pid; +#endif #elif defined(HAVE_PSET_INFO) processorid_t *cpuids; #endif }; +#if defined(__WIN32__) + +static __forceinline int +get_proc_affinity(erts_cpu_info_t *cpuinfo) +{ + DWORD pamask, samask; + if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) { + cpuinfo->cpuset = (cpu_set_t) pamask; + return 0; + } + else { + cpuinfo->cpuset = (cpu_set_t) 0; + return -erts_get_last_win_errno(); + } +} + +static __forceinline int +set_thr_affinity(cpu_set_t *set) +{ + if (*set == (cpu_set_t) 0) + return -ENOTSUP; + if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0) + return -erts_get_last_win_errno(); + else + return 0; +} + +#endif + erts_cpu_info_t * erts_cpu_info_create(void) { erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t)); if (!cpuinfo) return NULL; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) cpuinfo->affinity_str = NULL; +#if defined(HAVE_SCHED_xETAFFINITY) cpuinfo->pid = getpid(); +#endif #elif defined(HAVE_PSET_INFO) cpuinfo->cpuids = NULL; #endif @@ -162,10 +212,13 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo) #ifdef __WIN32__ { + int i; SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); cpuinfo->configured = (int) sys_info.dwNumberOfProcessors; - + for (i = 0; i < sizeof(DWORD)*8; i++) + if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i)) + cpuinfo->online++; } #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \ || defined(_SC_NPROCESSORS_ONLN)) @@ -205,8 +258,8 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo) if (cpuinfo->online > cpuinfo->configured) cpuinfo->online = cpuinfo->configured; -#ifdef HAVE_SCHED_xETAFFINITY - if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) { +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo) == 0) { int i, c, cn, si; c = cn = 0; si = sizeof(cpuinfo->affinity_str_buf) - 1; @@ -289,7 +342,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo) char * erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) if (!cpuinfo) return "false"; return cpuinfo->affinity_str; @@ -303,7 +356,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no) { if (!cpuinfo || no < 1 || cpuinfo->available < no) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t *allowed = &cpuinfo->cpuset; int ix, n; @@ -335,8 +388,8 @@ int erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id) { if (cpuinfo && 0 <= id) { -#ifdef HAVE_SCHED_xETAFFINITY - if (id <= CPU_SETSIZE) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (id < CPU_SETSIZE) return CPU_ISSET(id, &cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) int no; @@ -388,7 +441,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) */ if (!cpuinfo) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t bind_set; if (cpu < 0) @@ -398,9 +451,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) CPU_ZERO(&bind_set); CPU_SET(cpu, &bind_set); - if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&bind_set); } #elif defined(HAVE_PROCESSOR_BIND) if (cpu < 0) @@ -418,10 +469,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) { if (!cpuinfo) return -EINVAL; -#if defined(HAVE_SCHED_xETAFFINITY) - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0) - return -errno; - return 0; +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -434,7 +483,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) int erts_unbind_from_cpu_str(char *str) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *c = str; int cpus = 0; int shft = 0; @@ -486,9 +535,7 @@ erts_unbind_from_cpu_str(char *str) if (!cpus) return -EINVAL; - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -965,3 +1012,91 @@ read_topology(erts_cpu_info_t *cpuinfo) } #endif + +#if defined(__WIN32__) + +int +erts_get_last_win_errno(void) +{ + switch (GetLastError()) { + case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ + case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ + case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ + case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ + case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ + case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ + case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ + case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ + case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ + case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ + case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ + case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ + case ERROR_INVALID_DATA: return EINVAL; /* 13 */ + case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ + case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ + case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ + case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ + case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ + case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ + case ERROR_BAD_UNIT: return EACCES; /* 20 */ + case ERROR_NOT_READY: return EACCES; /* 21 */ + case ERROR_BAD_COMMAND: return EACCES; /* 22 */ + case ERROR_CRC: return EACCES; /* 23 */ + case ERROR_BAD_LENGTH: return EACCES; /* 24 */ + case ERROR_SEEK: return EACCES; /* 25 */ + case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ + case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ + case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ + case ERROR_WRITE_FAULT: return EACCES; /* 29 */ + case ERROR_READ_FAULT: return EACCES; /* 30 */ + case ERROR_GEN_FAILURE: return EACCES; /* 31 */ + case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ + case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ + case ERROR_WRONG_DISK: return EACCES; /* 34 */ + case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ + case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ + case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ + case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ + case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ + case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ + case ERROR_FAIL_I24: return EACCES; /* 83 */ + case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ + case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ + case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ + case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ + case ERROR_DISK_FULL: return ENOSPC; /* 112 */ + case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ + case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ + case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ + case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ + case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ + case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ + case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ + case ERROR_NOT_LOCKED: return EACCES; /* 158 */ + case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ + case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ + case ERROR_LOCK_FAILED: return EACCES; /* 167 */ + case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ + case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ + case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ + case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ + case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ + case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ + case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ + case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ + case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ + case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ + case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ + case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ + case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ + case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ + case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ + case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ + case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ + case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ + case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ + default: return EINVAL; + } +} + +#endif diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c new file mode 100644 index 0000000000..4db4cffd3a --- /dev/null +++ b/erts/lib_src/common/ethr_aux.c @@ -0,0 +1,762 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A Thread library for use in the ERTS and other OTP + * applications. + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <stdio.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_AUX_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" +#include <string.h> +#include <limits.h> + +#ifndef __WIN32__ +#include <unistd.h> +#endif + +#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100 +#define ERTS_TS_EV_ALLOC_POOL_SIZE 25 + +erts_cpu_info_t *ethr_cpu_info__; + +int ethr_not_completely_inited__ = 1; +int ethr_not_inited__ = 1; + +ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__; + +#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#endif + +void *(*ethr_thr_prepare_func__)(void) = NULL; +void (*ethr_thr_parent_func__)(void *) = NULL; +void (*ethr_thr_child_func__)(void *) = NULL; + +typedef struct ethr_xhndl_list_ ethr_xhndl_list; +struct ethr_xhndl_list_ { + ethr_xhndl_list *next; + void (*funcp)(void); +}; + +size_t ethr_pagesize__; +size_t ethr_min_stack_size__; /* kilo words */ +size_t ethr_max_stack_size__; /* kilo words */ + +ethr_rwmutex xhndl_rwmtx; +ethr_xhndl_list *xhndl_list; + +static int main_threads; + +static int init_ts_event_alloc(void); + +int +ethr_init_common__(ethr_init_data *id) +{ + int res; + if (id) { + ethr_thr_prepare_func__ = id->thread_create_prepare_func; + ethr_thr_parent_func__ = id->thread_create_parent_func; + ethr_thr_child_func__ = id->thread_create_child_func; + } + + ethr_cpu_info__ = erts_cpu_info_create(); + if (!ethr_cpu_info__) + return ENOMEM; + +#ifdef _SC_PAGESIZE + ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE); +#elif defined(HAVE_GETPAGESIZE) + ethr_pagesize__ = (size_t) getpagesize(); +#else + ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */ +#endif + + /* User needs at least 4 KB */ + ethr_min_stack_size__ = 4*1024; +#if SIZEOF_VOID_P == 8 + /* Double that on 64-bit archs */ + ethr_min_stack_size__ *= 2; +#endif + /* On some systems as much as about 4 KB is used by the system */ + ethr_min_stack_size__ += 4*1024; + /* There should be room for signal handlers */ +#ifdef SIGSTKSZ + ethr_min_stack_size__ += SIGSTKSZ; +#else + ethr_min_stack_size__ += ethr_pagesize__; +#endif + /* The system may think that we need more stack */ +#if defined(PTHREAD_STACK_MIN) + if (ethr_min_stack_size__ < PTHREAD_STACK_MIN) + ethr_min_stack_size__ = PTHREAD_STACK_MIN; +#elif defined(_SC_THREAD_STACK_MIN) + { + size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); + if (ethr_min_stack_size__ < thr_min_stk_sz) + ethr_min_stack_size__ = thr_min_stk_sz; + } +#endif + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ +#ifdef ETHR_STACK_GUARD_SIZE + ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE; +#endif + ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__); + + ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__); + + ethr_max_stack_size__ = 32*1024*1024; +#if SIZEOF_VOID_P == 8 + ethr_max_stack_size__ *= 2; +#endif + ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__); + +#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS + { + int i; + for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { + res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } + } +#endif + + res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__)); + if (res != 0) + return res; + + xhndl_list = NULL; + + return 0; +} + +int +ethr_late_init_common__(ethr_late_init_data *lid) +{ + ethr_ts_event *tsep = NULL; + int reader_groups; + int res; + int i; + ethr_memory_allocator *m[] = {ðr_mem__.std, + ðr_mem__.sl, + ðr_mem__.ll}; + if (lid) + ethr_mem__ = lid->mem; + if (!ethr_mem__.std.alloc + || !ethr_mem__.std.realloc + || !ethr_mem__.std.free) { + ethr_mem__.std.alloc = malloc; + ethr_mem__.std.realloc = realloc; + ethr_mem__.std.free = free; + } + for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) { + if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) { + m[i]->alloc = ethr_mem__.std.alloc; + m[i]->realloc = ethr_mem__.std.realloc; + m[i]->free = ethr_mem__.std.free; + } + + } + res = init_ts_event_alloc(); + if (res != 0) + return res; + res = ethr_make_ts_event__(&tsep); + if (res == 0) + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (!lid) { + main_threads = 0; + reader_groups = 0; + } + else { + if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads) + return res; + main_threads = lid->main_threads; + reader_groups = lid->reader_groups; + } + res = ethr_mutex_lib_late_init(reader_groups, main_threads); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; /* Need it for + rwmutex_init */ + res = ethr_rwmutex_init(&xhndl_rwmtx); + ethr_not_completely_inited__ = 1; + if (res != 0) + return res; + return 0; +} + +int +ethr_install_exit_handler(void (*funcp)(void)) +{ + ethr_xhndl_list *xhp; + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (!funcp) + return EINVAL; + + xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list)); + if (!xhp) + return ENOMEM; + + ethr_rwmutex_rwlock(&xhndl_rwmtx); + + xhp->funcp = funcp; + xhp->next = xhndl_list; + xhndl_list = xhp; + + ethr_rwmutex_rwunlock(&xhndl_rwmtx); + + return 0; +} + +void +ethr_run_exit_handlers__(void) +{ + ethr_xhndl_list *xhp; + + ethr_rwmutex_rlock(&xhndl_rwmtx); + + xhp = xhndl_list; + + ethr_rwmutex_runlock(&xhndl_rwmtx); + + for (; xhp; xhp = xhp->next) + (*xhp->funcp)(); +} + +/* + * Thread specific event alloc, etc. + * + * Note that we don't know when it is safe to destroy an event, but + * we know when it is safe to reuse it. ts_event_free() therefore + * never destroys an event (but makes freed events available for + * reuse). + * + * We could easily keep track of the usage of events, and by this + * make it possible to destroy events. We would however suffer a + * performance penalty for this and save very little memory. + */ + +typedef union { + ethr_ts_event ts_ev; + char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))]; +} ethr_aligned_ts_event; + +static ethr_spinlock_t ts_ev_alloc_lock; +static ethr_ts_event *free_ts_ev; + +#if SIZEOF_VOID_P == SIZEOF_INT +typedef unsigned int EthrPtrSzUInt; +#elif SIZEOF_VOID_P == SIZEOF_LONG +typedef unsigned long EthrPtrSzUInt; +#else +#error No pointer sized integer type +#endif + +static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp) +{ + int i; + ethr_aligned_ts_event *atsev; + atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size + + ETHR_CACHE_LINE_SIZE); + if (!atsev) + return NULL; + if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0) + atsev = ((ethr_aligned_ts_event *) + ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + for (i = 1; i < size; i++) { + atsev[i-1].ts_ev.next = &atsev[i].ts_ev; + ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0); + atsev[i-1].ts_ev.iflgs = 0; + } + ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0); + atsev[size-1].ts_ev.iflgs = 0; + atsev[size-1].ts_ev.next = NULL; + if (endpp) + *endpp = &atsev[size-1].ts_ev; + return &atsev[0].ts_ev; +} + +static int init_ts_event_alloc(void) +{ + free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE, + NULL); + if (!free_ts_ev) + return ENOMEM; + return ethr_spinlock_init(&ts_ev_alloc_lock); +} + +static ethr_ts_event *ts_event_alloc(void) +{ + ethr_ts_event *ts_ev; + ethr_spin_lock(&ts_ev_alloc_lock); + if (free_ts_ev) { + ts_ev = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + else { + ethr_ts_event *ts_ev_pool_end; + ethr_spin_unlock(&ts_ev_alloc_lock); + + ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end); + if (!ts_ev) + return NULL; + + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev_pool_end->next = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + return ts_ev; +} + +static void ts_event_free(ethr_ts_event *ts_ev) +{ + ETHR_ASSERT(!ts_ev->udata); + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev->next = free_ts_ev; + free_ts_ev = ts_ev; + ethr_spin_unlock(&ts_ev_alloc_lock); +} + +int ethr_make_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED; + tsep->udata = NULL; + tsep->rgix = 0; + tsep->mtix = 0; + + res = ethr_set_tse__(tsep); + if (res != 0 && tsepp && *tsepp) { + ts_event_free(tsep); + return res; + } + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP; + tsep->udata = NULL; + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_free_ts_event__(ethr_ts_event *tsep) +{ + ts_event_free(tsep); + return 0; +} + +void ethr_ts_event_destructor__(void *vtsep) +{ + if (vtsep) { + ethr_ts_event *tsep = (ethr_ts_event *) vtsep; + ts_event_free(tsep); + ethr_set_tse__(NULL); + } +} + +int ethr_set_main_thr_status(int on, int no) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + return EINVAL; + if (on) { + if (no < 1 || main_threads < no) + return EINVAL; + tsep->mtix = (unsigned short) no; + tsep->iflgs |= ETHR_TS_EV_MAIN_THR; + } + else { + tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR; + tsep->mtix = (unsigned short) 0; + } + return 0; +} + +int ethr_get_main_thr_status(int *on) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + *on = 0; + else { + if (tsep->iflgs & ETHR_TS_EV_MAIN_THR) + *on = 1; + else + *on = 0; + } + return 0; +} + + +/* Atomics */ + +void +ethr_atomic_init(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(var); + ethr_atomic_init__(var, i); +} + +void +ethr_atomic_set(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set__(var, i); +} + +long +ethr_atomic_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read__(var); +} + + +long +ethr_atomic_add_read(ethr_atomic_t *var, long incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_add_read__(var, incr); +} + +long +ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read__(var); +} + +long +ethr_atomic_dec_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read__(var); +} + +void +ethr_atomic_add(ethr_atomic_t *var, long incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_add__(var, incr); +} + +void +ethr_atomic_inc(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_inc__(var); +} + +void +ethr_atomic_dec(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec__(var); +} + +long +ethr_atomic_read_band(ethr_atomic_t *var, long mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_band__(var, mask); +} + +long +ethr_atomic_read_bor(ethr_atomic_t *var, long mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_bor__(var, mask); +} + +long +ethr_atomic_xchg(ethr_atomic_t *var, long new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_xchg__(var, new); +} + +long +ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg__(var, new, expected); +} + +long +ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_acqb__(var); +} + +long +ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read_acqb__(var); +} + +void +ethr_atomic_set_relb(ethr_atomic_t *var, long i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set_relb__(var, i); +} + +void +ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec_relb__(var); +} + +long +ethr_atomic_dec_read_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read_relb__(var); +} + +long +ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_acqb__(var, new, exp); +} + +long +ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_relb__(var, new, exp); +} + + +/* Spinlocks and rwspinlocks */ + +int +ethr_spinlock_init(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_init__(lock); +} + +int +ethr_spinlock_destroy(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_destroy__(lock); +} + +void +ethr_spin_unlock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_unlock__(lock); +} + +void +ethr_spin_lock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_lock__(lock); +} + +int +ethr_rwlock_init(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_init__(lock); +} + +int +ethr_rwlock_destroy(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_destroy__(lock); +} + +void +ethr_read_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_unlock__(lock); +} + +void +ethr_read_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_lock__(lock); +} + +void +ethr_write_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_unlock__(lock); +} + +void +ethr_write_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_lock__(lock); +} + +ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err) +{ + char *errstr; + if (err == ENOTSUP) + errstr = "Operation not supported"; + else { + errstr = strerror(err); + if (!errstr) + errstr = "Unknown error"; + } + fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n", + file, line, func, errstr, err); + ethr_abort__(); +} + +int ethr_assert_failed(const char *file, int line, const char *func, char *a) +{ + fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a); + ethr_abort__(); + return 0; +} diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c new file mode 100644 index 0000000000..04feceec89 --- /dev/null +++ b/erts/lib_src/common/ethr_cbf.c @@ -0,0 +1,36 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +/* + * We keep this function alone in a separate file so the + * compiler wont optimize it away. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ethread.h" + +void +ethr_compiler_barrier_fallback(void) +{ + +} diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c new file mode 100644 index 0000000000..f918bba81d --- /dev/null +++ b/erts/lib_src/common/ethr_mutex.c @@ -0,0 +1,2758 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_MUTEX_IMPL__ + +#include <limits.h> +#include "ethread.h" +#include "ethr_internal.h" + +#define ETHR_SPIN_WITH_WAITERS 1 + +#define ETHR_MTX_MAX_FLGS_SPIN 1000 + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static int default_rwmtx_main_spincount; +static int default_rwmtx_aux_spincount; +#endif +#ifdef ETHR_USE_OWN_MTX_IMPL__ +static int default_mtx_main_spincount; +static int default_mtx_aux_spincount; +static int default_cnd_main_spincount; +static int default_cnd_aux_spincount; +#endif + +static int no_spin; + +#ifndef ETHR_USE_OWN_RWMTX_IMPL__ +static pthread_rwlockattr_t write_pref_attr_data; +static pthread_rwlockattr_t *write_pref_attr; +#endif + +#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__) +# define ETHR_MTX_QLOCK_INIT ethr_spinlock_init +# define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy +# define ETHR_MTX_Q_LOCK ethr_spin_lock +# define ETHR_MTX_Q_UNLOCK ethr_spin_unlock +#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__) +# define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL) +# define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy +# define ETHR_MTX_Q_LOCK(L) \ +do { \ + int res__ = pthread_mutex_lock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +# define ETHR_MTX_Q_UNLOCK(L) \ +do { \ + int res__ = pthread_mutex_unlock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__) +# define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0) +# define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0) +# define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL)) +# define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL)) +#endif + +int +ethr_mutex_lib_init(int cpu_conf) +{ + int res = 0; + + no_spin = cpu_conf == 1; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT; + default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT; + default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT; + default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT; +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT; + default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT; + +#else + +#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \ + && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) + res = pthread_rwlockattr_init(&write_pref_attr_data); + if (res != 0) + return res; + res = pthread_rwlockattr_setkind_np( + &write_pref_attr_data, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + write_pref_attr = &write_pref_attr_data; +#else + write_pref_attr = NULL; +#endif + +#endif + + return res; +} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS +#if 0 /* + * When inc and dec are real atomic instructions as on x86, the + * ETHR_RLOCK_WITH_INC_DEC implementations performs better with + * lots of read locks compared to the cmpxchg based implementation. + * It, however, performs worse with lots of mixed reads and writes. + * It could be used for rwlocks that are known to be read locked + * much, but the readers array based implementation outperforms it + * by far. Therefore, it has been disabled, and will probably be + * removed some time in the future. + */ +# define ETHR_RLOCK_WITH_INC_DEC +#endif +#endif + +static int reader_groups_array_size = 0; +static int main_threads_array_size = 0; + +#endif + +int +ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads) +{ +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + reader_groups_array_size = (no_reader_groups <= 1 + ? 1 + : no_reader_groups + 1); + main_threads_array_size = (no_main_threads <= 1 + ? 1 + : no_main_threads + 1); +#endif + return 0; +} + +int +ethr_rwmutex_set_reader_group(int ix) +{ +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + ethr_ts_event *tse; + + if (ix < 0 || reader_groups_array_size <= ix) + return EINVAL; + + tse = ethr_get_ts_event(); + + if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) { + ethr_leave_ts_event(tse); + return EINVAL; + } + + tse->rgix = ix; + + ethr_leave_ts_event(tse); +#endif + return 0; +} + +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) +static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int); +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0) +#else +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + long initial); +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + long initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock); +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */ + +static ETHR_INLINE void +rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? tse->rgix + : tse->mtix); + rwmtx->tdata.ra[ix].data.waiting_readers++; +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx, + ethr_rwmutex_type type, + int ix, + int inc) +{ + if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0) + ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc); + else { + ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ETHR_ASSERT(inc == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + } +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_inc: + ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_inc; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + } +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec: + ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + return (long) 0; + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + return (long) 0; + } +} + +static ETHR_INLINE long +rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) +{ + long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers); +#ifdef ETHR_DEBUG + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + ETHR_ASSERT(res >= 0); + break; + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + ETHR_ASSERT(res == 0 || res == 1); + break; + default: + ETHR_ASSERT(0); + break; + } +#endif + return res; +} + + +static ETHR_INLINE void +enqueue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (!*queue) { + *queue = tse_start; + tse_start->prev = tse_end; + tse_end->next = tse_start; + } + else { + tse_end->next = *queue; + tse_start->prev = (*queue)->prev; + (*queue)->prev->next = tse_start; + (*queue)->prev = tse_end; + } +} + +static ETHR_INLINE void +insert(ethr_ts_event *tse_pred, ethr_ts_event *tse) +{ + tse->next = tse_pred->next; + tse->prev = tse_pred; + tse_pred->next->prev = tse; + tse_pred->next = tse; +} + +static ETHR_INLINE void +dequeue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (tse_start->prev == tse_end) { + ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start); + *queue = NULL; + } + else { + if (*queue == tse_start) + *queue = tse_end->next; + tse_end->next->prev = tse_start->prev; + tse_start->prev->next = tse_end->next; + } +} + +static void +event_wait(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int spincount, + long type, + int is_rwmtx, + int is_freq_read) +{ + int locked = 0; + long act; + int need_try_complete_runlock = 0; + + /* Need to enqueue and wait... */ + + tse->uflgs = type; + ethr_atomic_set(&tse->uaflgs, type); + + ETHR_MTX_Q_LOCK(&mtxb->qlck); + locked = 1; + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + act = ethr_atomic_read(&mtxb->flgs); + + if (act & type) { + + /* Wait bit already there; enqueue... */ + + ETHR_ASSERT(mtxb->q); + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + enqueue(&mtxb->q, tse, tse); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws++; +#endif + } + else { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(rwmtx->rq_end); + insert(rwmtx->rq_end, tse); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } + } + else { + + /* Set wait bit */ + + while (1) { + long new, exp = act; + int freqread_tryrlock = 0; + need_try_complete_runlock = 0; + + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) + need_try_complete_runlock = 1; + if (act != 0) + new = act | ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; /* Try to get it */ + } + else { + ETHR_ASSERT(is_rwmtx); + + if (!is_freq_read) { + if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__)) + new = act | ETHR_RWMTX_R_WAIT_FLG__; + else + new = act + 1; /* Try to get it */ + } + else { + if (act & ~ETHR_RWMTX_R_FLG__) + new = act | ETHR_RWMTX_R_WAIT_FLG__; + else { /* Try to get it */ + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + rwmutex_freqread_rdrs_inc(rwmtx, tse); + ETHR_MEMORY_BARRIER; + new = act | ETHR_RWMTX_R_FLG__; + freqread_tryrlock = 1; + } + } + } + + act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp); + if (exp == act) { + if (new & type) { + act = new; + break; + } + else { + /* Got it */ + goto done; + } + } + + if (freqread_tryrlock) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + + /* We didn't set ETHR_RWMTX_R_FLG__, however someone + else might have */ + if (act == ETHR_RWMTX_R_FLG__) + goto done; /* Got it by help from someone else */ + + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); + /* + * We know that no waiter flags have been set, i.e., + * we cannot get into a situation where we need to wake + * someone up here. Just restore the readers counter + * and do it over again... + */ + rwmutex_freqread_rdrs_dec(rwmtx, tse); + } + } + + /* Enqueue */ + + if (type == ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(!rwmtx->rq_end); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + else { + mtxb->ws++; + } +#endif + + enqueue(&mtxb->q, tse, tse); + } + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + /* Wait */ + locked = 0; + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + if (need_try_complete_runlock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We were the only one in queue when we enqueued, and it + * was seemingly read locked. We need to try to complete a + * runlock otherwise we might be hanging forever. If the + * runlock could be completed we will be dequeued and + * woken by ourselves. + */ + rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, + act, tse, 0, 1, 0); + } + + while (1) { + ethr_event_reset(&tse->event); + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + + ETHR_ASSERT(act == type); + ethr_event_swait(&tse->event, spincount); + /* swait result: 0 || EINTR */ + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + } + + done: + if (locked) + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); +} + +static void +wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + ethr_ts_event *tse; + + tse = mtxb->q; + ETHR_ASSERT(tse); + dequeue(&mtxb->q, tse, tse); + + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws--; +#endif +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + ethr_atomic_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); +} + +static ETHR_INLINE int +initial_spincount(struct ethr_mutex_base_ *mtxb) +{ + return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN + ? mtxb->aux_scnt + : ETHR_MTX_MAX_FLGS_SPIN); +} + +static ETHR_INLINE int +update_spincount(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int *start_scnt, + int *scnt) +{ + int sscnt = *start_scnt; + if (sscnt < 0) { + *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? mtxb->main_scnt + : mtxb->aux_scnt); + *scnt -= ETHR_MTX_MAX_FLGS_SPIN; + } + else { + *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? mtxb->main_scnt + : mtxb->aux_scnt); + *scnt -= sscnt; + if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) { + *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt; + *start_scnt = -1; + return 0; + } + } + return 1; +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length, + int pre_check); + +static ETHR_INLINE void +write_lock_wait(struct ethr_mutex_base_ *mtxb, + long initial, + int is_rwmtx, + int is_freq_read) +{ + long act = initial; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + int res; + int freq_read_size = -1; + int freq_read_start_ix = -1; + + ETHR_ASSERT(!is_freq_read || is_rwmtx); + + start_scnt = scnt = initial_spincount(mtxb); + + /* + * Spin trying to write lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + long exp; + + while (act != 0) { + + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + if (!tse) + tse = ethr_get_ts_event(); + if (freq_read_size < 0) { + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + freq_read_size = reader_groups_array_size; + freq_read_start_ix = tse->rgix; + } + else { + freq_read_size = main_threads_array_size; + freq_read_start_ix = tse->mtix; + } + } + res = check_readers_array(rwmtx, + freq_read_start_ix, + freq_read_size, + 1); + scnt--; + if (res == 0) { + act = ethr_atomic_read(&mtxb->flgs); + if (act & ETHR_RWMTX_R_MASK__) { + res = rwmutex_try_complete_runlock(rwmtx, act, + tse, 0, 0, + 1); + if (res != EBUSY) + goto done; /* Got it */ + } + if (scnt <= 0) + goto chk_spin; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + continue; + } + } + + if (scnt <= 0) { + chk_spin: + scnt = 0; + + if (!tse) + tse = ethr_get_ts_event(); + if (update_spincount(mtxb, tse, &start_scnt, &scnt)) { + event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__, + is_rwmtx, is_freq_read); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&mtxb->flgs); + scnt--; + } + ETHR_ASSERT(scnt >= 0); + + exp = act; + + act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, + ETHR_RWMTX_W_FLG__, + exp); + if (act == 0) + goto done; /* Got it */ + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static int +mtxb_init(struct ethr_mutex_base_ *mtxb, + int def_main_scnt, + int main_scnt, + int def_aux_scnt, + int aux_scnt) +{ + ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws = 0; +#endif + if (no_spin) { + mtxb->main_scnt = 0; + mtxb->aux_scnt = 0; + } + else { + if (main_scnt > SHRT_MAX) + mtxb->main_scnt = SHRT_MAX; + else if (main_scnt < 0) + mtxb->main_scnt = def_main_scnt; + else + mtxb->main_scnt = (short) main_scnt; + if (aux_scnt > SHRT_MAX) + mtxb->aux_scnt = SHRT_MAX; + else if (aux_scnt < 0) + mtxb->aux_scnt = def_aux_scnt; + else + mtxb->aux_scnt = (short) aux_scnt; + if (mtxb->main_scnt < mtxb->aux_scnt) + mtxb->main_scnt = mtxb->aux_scnt; + + } + mtxb->q = NULL; + ethr_atomic_init(&mtxb->flgs, 0); + return ETHR_MTX_QLOCK_INIT(&mtxb->qlck); +} + +static int +mtxb_destroy(struct ethr_mutex_base_ *mtxb) +{ + long act; + ETHR_MTX_Q_LOCK(&mtxb->qlck); + act = ethr_atomic_read(&mtxb->flgs); + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + if (act != 0) + return EINVAL; + return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck); +} + + +#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Mutex and condition variable implementation * +\* */ + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +/* -- Mutex ---------------------------------------------------------------- */ + +int +ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt) +{ + int res; +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx); + res = mtxb_init(&mtx->mtxb, + default_mtx_main_spincount, + opt ? opt->main_spincount : -1, + default_mtx_aux_spincount, + opt ? opt->aux_spincount : -1); +#if ETHR_XCHK + if (res != 0) + mtx->initialized = 0; +#endif + return res; +} + +int +ethr_mutex_init(ethr_mutex *mtx) +{ + return ethr_mutex_init_opt(mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = 0; +#endif + return mtxb_destroy(&mtx->mtxb); +} + +void +ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial) +{ + write_lock_wait(&mtx->mtxb, initial, 0, 0); +} + +void +ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) +{ + ethr_ts_event *tse; + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + tse = mtx->mtxb.q; + + ETHR_ASSERT(tse); + ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs) + == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__); + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + + /* + * If we have multiple waiters, there is no need to modify + * mtxb->flgs; otherwise, we need to clear the write wait bit... + */ + if (tse->next == mtx->mtxb.q) + ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); + + wake_writer(&mtx->mtxb, 0); +} + +/* -- Condition variables -------------------------------------------------- */ + +static void +enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) +{ + long act; + + /* + * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx' + * is not currently locked by current thread, we almost certainly have a + * hard to debug race condition. There might however be some (strange) + * use for it. POSIX also allow a call to `pthread_cond_signal' or + * `pthread_cond_broadcast' even though the the associated mutex isn't + * locked by the caller. Therefore, we also allow this kind of strange + * usage, but optimize for the case where the mutex is locked by the + * calling thread. + */ + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + { + int dbg_nws__ = 0; + ethr_ts_event *dbg_tse__; + for (dbg_tse__ = tse_start; + dbg_tse__ != tse_end; + dbg_tse__ = dbg_tse__->next) + dbg_nws__++; + mtx->mtxb.ws += dbg_nws__ + 1; + } +#endif + + act = ethr_atomic_read(&mtx->mtxb.flgs); + ETHR_ASSERT(act == 0 + || act == ETHR_RWMTX_W_FLG__ + || act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + if (act & ETHR_RWMTX_W_FLG__) { + /* The normal sane case */ + if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) { + ETHR_ASSERT(!mtx->mtxb.q); + act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, + (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__), + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) { + /* + * Sigh... this wasn't so sane after all since, the mutex was + * obviously not locked by the current thread.... + */ + ETHR_ASSERT(act == 0); + goto mtx_unlocked; + } + } + +#ifdef ETHR_DEBUG + if (act & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_ASSERT(mtx->mtxb.q); + else + ETHR_ASSERT(!mtx->mtxb.q); +#endif + + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + int multi; + mtx_unlocked: + /* Sigh... mutex isn't locked... */ + + multi = tse_start != tse_end; + + while (1) { + long new, exp = act; + + if (multi || (act & ETHR_RWMTX_W_FLG__)) + new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; + + act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp); + if (exp == act) { + ETHR_ASSERT(!mtx->mtxb.q); + if (act & ETHR_RWMTX_W_FLG__) { + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + ETHR_ASSERT(!mtx->mtxb.q); + /* + * Acquired the mutex on behalf of the + * first thread in the queue; wake + * it and enqueue the rest... + */ +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtx->mtxb.ws--; +#endif + if (multi) { + enqueue(&mtx->mtxb.q, tse_start->next, tse_end); + ETHR_ASSERT(mtx->mtxb.q); + } + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + ethr_atomic_set(&tse_start->uaflgs, 0); + ethr_event_set(&tse_start->event); + } + break; + } + } + } +} + +int +ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) +{ + int res; +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd); + cnd->q = NULL; + if (no_spin) { + cnd->main_scnt = 0; + cnd->aux_scnt = 0; + } + else { + if (!opt || opt->main_spincount < 0) + cnd->main_scnt = default_cnd_main_spincount; + else if (opt->main_spincount > SHRT_MAX) + cnd->main_scnt = SHRT_MAX; + else + cnd->main_scnt = (short) opt->main_spincount; + if (!opt || opt->aux_spincount < 0) + cnd->aux_scnt = default_cnd_aux_spincount; + else if (opt->aux_spincount > SHRT_MAX) + cnd->aux_scnt = SHRT_MAX; + else + cnd->aux_scnt = (short) opt->aux_spincount; + if (cnd->main_scnt < cnd->aux_scnt) + cnd->main_scnt = cnd->aux_scnt; + } + ETHR_MTX_QLOCK_INIT(&cnd->qlck); + return 0; +} + +int +ethr_cond_init(ethr_cond *cnd) +{ + return ethr_cond_init_opt(cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__); + + ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + tse->next = tse->prev = NULL; + + enqueue_mtx(mtx, tse, tse); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + } +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int res; + int got_all; + ethr_ts_event *tse; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + do { + got_all = 1; + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + ethr_ts_event *tse_tmp, *tse_end; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = cnd->q->prev; + + tse_tmp = tse; + + do { + + if (mtx == (ethr_mutex *) tse_tmp->udata) { + /* The normal case */ + + ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs) + == ETHR_CND_WAIT_FLG__); + + ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + } + else { + /* Should be very unusual */ + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = tse_tmp->prev; + got_all = 0; + break; + } + + tse_tmp = tse_tmp->next; + + } while (tse_tmp != cnd->q); + + dequeue(&cnd->q, tse, tse_end); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + enqueue_mtx(mtx, tse, tse_end); + } + + } while (!got_all); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int woken; + int scnt; + int res; + void *udata = NULL; + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(!cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(!mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + tse = ethr_get_ts_event(); + + scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? cnd->main_scnt + : cnd->aux_scnt); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + udata = tse->udata; /* Got to restore udata before returning */ + tse->udata = (void *) mtx; + + tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */ + ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + enqueue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + ethr_mutex_unlock(mtx); + + /* Wait */ + woken = 0; + while (1) { + long act; + + ethr_event_reset(&tse->event); + + act = ethr_atomic_read_acqb(&tse->uaflgs); + if (!act) + break; /* Mtx locked */ + + /* First time, got EINTR, or spurious wakeup... */ + + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + + if (woken) { + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued + * on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) { + ETHR_MTX_Q_LOCK(&cnd->qlck); + act = ethr_atomic_read(&tse->uaflgs); + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already + * enqueued on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + if (act == ETHR_CND_WAIT_FLG__) { + tse->udata = udata; + ethr_leave_ts_event(tse); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ethr_mutex_lock(mtx); + return EINTR; + } + } + ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__); + } + ethr_event_swait(&tse->event, scnt); + /* swait result: 0 || EINTR */ + woken = 1; + } + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + tse->udata = udata; + ethr_leave_ts_event(tse); + return 0; +} + +#else +/* -- pthread mutex and condition variables -------------------------------- */ + +int +ethr_mutex_init(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + return pthread_mutex_init(&mtx->pt_mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif +#if ETHR_XCHK + mtx->initialized = 0; +#endif + return pthread_mutex_destroy(&mtx->pt_mtx); +} + +int +ethr_cond_init(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + return pthread_cond_init(&cnd->pt_cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return pthread_cond_destroy(&cnd->pt_cnd); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_signal(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_broadcast(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); + if (res != 0 && res != EINTR) + ETHR_FATAL_ERROR__(res); + return res; +} + +#endif /* pthread_mutex */ + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_mutex_trylock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + return ethr_mutex_trylock__(mtx); +} + +void +ethr_mutex_lock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_lock__(mtx); +} + +void +ethr_mutex_unlock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_unlock__(mtx); +} + + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Read/Write Mutex * +\* */ + +static void +wake_readers(ethr_rwmutex *rwmtx, int rs) +{ + ethr_ts_event *tse; +#ifdef ETHR_DEBUG + int drs = 0; +#endif + + tse = rwmtx->mtxb.q; + ETHR_ASSERT(tse); + ETHR_ASSERT(rwmtx->rq_end); + dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end); + rwmtx->rq_end->next = NULL; + rwmtx->rq_end = NULL; + + ETHR_ASSERT(!rwmtx->mtxb.q + || (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs) + == ETHR_RWMTX_W_WAIT_FLG__)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + + while (tse) { + ethr_ts_event *tse_next; + +#ifdef ETHR_DEBUG + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) + == ETHR_RWMTX_R_WAIT_FLG__); + drs++; +#endif + + tse_next = tse->next; /* we aren't allowed to read tse->next + after we have reset uaflgs */ + + ethr_atomic_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); + tse = tse_next; + } + + ETHR_ASSERT(rs == drs); +} + +static ETHR_INLINE int +is_w_waiter(ethr_ts_event *tse) +{ + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__ + || tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__; +} + +static ETHR_INLINE int +multiple_w_waiters(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(rwmtx->mtxb.q); + ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + + if (!rwmtx->rq_end) + return rwmtx->mtxb.q->next != rwmtx->mtxb.q; + else { + ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q); + if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__) + return 1; + ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q + || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + return rwmtx->rq_end->next != rwmtx->mtxb.q; + } +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length, + int pre_check) +{ + int ix = start_rix; + +#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL + if (pre_check) + ETHR_READ_MEMORY_BARRIER; + else +#endif + ETHR_MEMORY_BARRIER; + + do { + long act = rwmutex_freqread_rdrs_read(rwmtx, ix); + if (act != 0) + return EBUSY; + ix++; + if (ix == length) + ix = 0; + } while (ix != start_rix); + + return 0; +} + +static ETHR_INLINE void +rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + long act; + /* + * Restore failed increment + */ + act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse); + + ETHR_WRITE_MEMORY_BARRIER; + + if (act == 0) { + +#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL + ETHR_READ_MEMORY_BARRIER; +#endif + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if ((act & ETHR_RWMTX_W_FLG__) == 0 + && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) { + /* + * We either got waiters, or someone else trying + * to read unlock which we might have to help. + */ + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0); + } + } +} + +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + long initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock) +{ + ethr_ts_event *tse_tmp; + long act = initial; + int six, res, length; + + tse_tmp = tse; + if (!tse_tmp) + tse_tmp = ethr_get_ts_event(); + + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) + goto check_waiters; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + length = reader_groups_array_size; + six = tse_tmp->rgix; + } + else { + length = main_threads_array_size; + six = tse_tmp->mtix; + } + if (start_next_ix) { + six++; + if (six >= length) + six = 0; + } + + if (!tse) + ethr_leave_ts_event(tse_tmp); + + if (check_before_try) { + res = check_readers_array(rwmtx, six, length, 1); + if (res == EBUSY) + return try_write_lock ? EBUSY : 0; + } + + while (1) { + long exp = act; + long new = act+1; + + ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + < ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + break; + } + if (!try_write_lock) { + if (act == ETHR_RWMTX_W_FLG__ || act == 0) + return 0; + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + return 0; + } + else if ((act & ETHR_RWMTX_R_FLG__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + return 0; + goto check_waiters; + } + } + else { + if (act == 0) + goto tryrwlock; + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) + return EBUSY; + } + } + + res = check_readers_array(rwmtx, six, length, 0); + if (res == EBUSY) { + act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); + if (act & ETHR_RWMTX_R_MASK__) + return try_write_lock ? EBUSY : 0; + } + else { + while (1) { + long exp = act; + long new = act; + new &= ~ETHR_RWMTX_R_FLG__; + new--; + + ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + return try_write_lock ? EBUSY : 0; + act = new; + break; + } + } + } + + /* + * Read unlock completed, but we have to check if + * threads have to be woken (or if we should try + * to write lock it). + */ + + if (act & ETHR_RWMTX_W_FLG__) + return try_write_lock ? EBUSY : 0; + + if (act & ETHR_RWMTX_WAIT_FLGS__) { + check_waiters: + rwmutex_unlock_wake(rwmtx, 0, act); + return try_write_lock ? EBUSY : 0; + } + + if (!try_write_lock) + return 0; + + tryrwlock: + /* Try to write lock it */ + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, + 0); + return act == 0 ? 0 : EBUSY; +} + +#ifdef ETHR_RLOCK_WITH_INC_DEC + +static ETHR_INLINE void +rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx) +{ + long act; + /* + * Restore failed increment + */ + act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + rwmutex_unlock_wake(rwmtx, 0, act); + } +} + +#endif + +static void +rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, + long initial) +{ + long act = initial, exp; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + +#ifdef ETHR_RLOCK_WITH_INC_DEC + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + act = ethr_atomic_read(&rwmtx->mtxb.flgs); +#endif + + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt == 0) { + tse = ethr_get_ts_event(); + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 0); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + scnt--; + } + exp = act; + ETHR_ASSERT(scnt >= 0); + +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs); + if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0) + goto done; /* Got it */ +#else + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) + goto done; /* Got it */ +#endif + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse, + long initial) +{ + long act = initial; + int scnt, start_scnt; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + if (scnt == 0) { + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 1); + return; /* Got it */ + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + scnt--; + } + + ETHR_ASSERT(scnt >= 0); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if (act == ETHR_RWMTX_R_FLG__) + return; /* Got it */ + + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) + break; /* Busy (need to restore inc) */ + + if (act & ETHR_RWMTX_R_FLG__) + return; /* Got it */ + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; /* Got it */ + } + } +} + +static void +rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 0); +} + +static void +rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 1); +} + +static ETHR_INLINE void +rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial) +{ + long act, act_mask; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + /* r pend unlock mask may vary and must be retained */ + act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + } + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + /* rs mask may vary and must be retained */ + act_mask = ETHR_RWMTX_RS_MASK__; +#else + /* rs mask always zero */ + ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0); + ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial); + return; +#endif + } + + act = act_initial; + while (1) { + long exp = act; + long new = new_initial + (act & act_mask); + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +#ifdef ETHR_DEBUG + +static void +dbg_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + ethr_ts_event *tse) +{ + long exp, act, imask; + + exp = have_w ? ETHR_RWMTX_W_FLG__ : 0; + + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + imask = ETHR_RWMTX_RS_MASK__; +#else + imask = 0; +#endif + } + + ETHR_ASSERT(tse); + + if (is_w_waiter(tse)) { + + exp |= ETHR_RWMTX_W_WAIT_FLG__; + if (rwmtx->rq_end) { + exp |= ETHR_RWMTX_R_WAIT_FLG__; + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } + else { + + exp |= ETHR_RWMTX_R_WAIT_FLG__; + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + exp |= ETHR_RWMTX_W_WAIT_FLG__; + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } +} + +#endif + +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) +{ + long new, act = initial; + ethr_ts_event *tse; + + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (!have_w) + return; + else { + while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + long exp = act; + new = exp & ~ETHR_RWMTX_W_FLG__; + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; + } + } + } + + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + tse = rwmtx->mtxb.q; + + if (!have_w) { + if (!tse) { +#ifdef ETHR_DEBUG + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); +#endif + goto already_served; + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act & ~ETHR_RWMTX_WAIT_FLGS__) { + already_served: + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } + } + +#ifdef ETHR_DEBUG + dbg_unlock_wake(rwmtx, have_w, tse); +#endif + + if (is_w_waiter(tse)) { + + if (!have_w) { + act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__); + ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0); + ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__); + act |= ETHR_RWMTX_W_FLG__; + } + + /* + * If we have multiple write waiters, there + * is no need to modify mtxb->flgs; otherwise, + * we need to clear the write wait bit... + */ + if (!multiple_w_waiters(rwmtx)) { + new = ETHR_RWMTX_W_FLG__; + if (tse->next != rwmtx->mtxb.q) { + ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + new |= ETHR_RWMTX_R_WAIT_FLG__; + } + + rwlock_wake_set_flags(rwmtx, new, act); + } + + wake_writer(&rwmtx->mtxb, 1); + } + else { + int rs; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) { + rs = rwmtx->tdata.rs; + new = (long) rs; + rwmtx->tdata.rs = 0; + } + else { + ethr_rwmutex_type type = rwmtx->type; + int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? reader_groups_array_size + : main_threads_array_size); + int ix; + + rs = 0; + for (ix = 0; ix < length; ix++) { + int wrs = rwmtx->tdata.ra[ix].data.waiting_readers; + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + ETHR_ASSERT(wrs >= 0); + if (wrs) { + rs += wrs; + rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs); + } + } + new = ETHR_RWMTX_R_FLG__; + } + + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + new |= ETHR_RWMTX_W_WAIT_FLG__; + + rwlock_wake_set_flags(rwmtx, new, act); + wake_readers(rwmtx, rs); + } +} + +static ethr_rwmtx_readers_array__ * +alloc_readers_array(int length, ethr_rwmutex_lived lived) +{ + ethr_rwmtx_readers_array__ *ra; + size_t sz; + void *mem; + + sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1); + + switch (lived) { + case ETHR_RWMUTEX_LONG_LIVED: + mem = ethr_mem__.ll.alloc(sz); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + mem = ethr_mem__.sl.alloc(sz); + break; + default: + mem = ethr_mem__.std.alloc(sz); + break; + } + if (!mem) + return NULL; + + if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) { + ra = (ethr_rwmtx_readers_array__ *) mem; + ra->data.byte_offset = 0; + } + else { + ra = ((ethr_rwmtx_readers_array__ *) + ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + ra->data.byte_offset = (int) ((unsigned long) ra + - (unsigned long) mem); + } + ra->data.lived = lived; + return ra; +} + +static void +free_readers_array(ethr_rwmtx_readers_array__ *ra) +{ + void *ptr = (void *) (((char *) ra) - ra->data.byte_offset); + switch (ra->data.lived) { + case ETHR_RWMUTEX_LONG_LIVED: + ethr_mem__.ll.free(ptr); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + ethr_mem__.sl.free(ptr); + break; + default: + ethr_mem__.std.free(ptr); + break; + } +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + int res; + ethr_rwmtx_readers_array__ *ra = NULL; +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx); + rwmtx->rq_end = NULL; + rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL; + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + if (main_threads_array_size <= reader_groups_array_size) { + /* No point using reader groups... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ; + } + /* Fall through */ + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + int length; + + length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ + ? main_threads_array_size + : reader_groups_array_size); + + if (length == 1) { + /* No point using a frequent reader type... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL; + } + else { + int ix; + ra = alloc_readers_array(length, + (opt + ? opt->lived + : ETHR_RWMUTEX_UNKNOWN_LIVED)); + if (!ra) { + res = ENOMEM; + goto error; + } + + rwmtx->tdata.ra = ra; + + for (ix = 0; ix < length; ix++) { + ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0); + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + } + break; + } + } + case ETHR_RWMUTEX_TYPE_NORMAL: + rwmtx->tdata.rs = 0; + break; + default: + res = EINVAL; + goto error; + } + res = mtxb_init(&rwmtx->mtxb, + default_rwmtx_main_spincount, + opt ? opt->main_spincount : -1, + default_rwmtx_aux_spincount, + opt ? opt->aux_spincount : -1); + if (res == 0) + return 0; + + error: + + if (ra) + free_readers_array(ra); + +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ + return ethr_rwmutex_init_opt(rwmtx, NULL); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + long act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act == ETHR_RWMTX_R_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0); + } + res = mtxb_destroy(&rwmtx->mtxb); + if (res != 0) + return res; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + free_readers_array(rwmtx->tdata.ra); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return 0; +} + +#define ETHR_MAX_TRYRLOCK_TRIES 5 + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + res = EBUSY; + else { + act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + res = EBUSY; + } + } +#else + long exp = 0; + int tries = 0; + + while (1) { + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) { + res = 0; + break; + } + if (tries > ETHR_MAX_TRYRLOCK_TRIES + || (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) { + res = EBUSY; + break; + } + tries++; + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + res = EBUSY; + break; + } + + if (act & ETHR_RWMTX_R_FLG__) + break; + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + return res; +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + rwmutex_normal_rlock_wait(rwmtx, act); +#else + long exp = 0; + + while (1) { + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) { + break; + } + + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_normal_rlock_wait(rwmtx, act); + break; + } + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + while (1) { + long exp, new; + + if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { + rwmutex_freqread_rlock_wait(rwmtx, tse, act); + break; + } + + if (act & ETHR_RWMTX_R_FLG__) + break; + + exp = act; + new = act | ETHR_RWMTX_R_FLG__; + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + rwmutex_unlock_wake(rwmtx, 0, act); + } + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse); + + ETHR_ASSERT(act >= 0); + + ETHR_WRITE_MEMORY_BARRIER; + + if (act == 0) { + +#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL + ETHR_READ_MEMORY_BARRIER; +#endif + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + if ((act & ETHR_RWMTX_W_FLG__) == 0 + && (act & (ETHR_RWMTX_WAIT_FLGS__ + | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) { + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + long act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + res = EBUSY; + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + res = 0; + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + do { + + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) { + res = EBUSY; + break; + } + + if (act & ETHR_RWMTX_R_MASK__) { + res = rwmutex_try_complete_runlock(rwmtx, act, NULL, + 0, 1, 1); + break; + } + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + + } while (act != 0); + + break; + } + + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + return res; +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + long act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + rwmutex_normal_rwlock_wait(rwmtx, act); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + + do { + + if (act != 0) { + rwmutex_freqread_rwlock_wait(rwmtx, act); + break; + } + + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + + } while (act != 0); + + break; + } + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + long act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, + 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act); + break; + } + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +#else +/* -- pthread read/write mutex --------------------------------------------- */ + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ +#if ETHR_XCHK + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr); +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + return ethr_rwmutex_init(rwmtx); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrlock__(rwmtx); +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rlock__(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_runlock__(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrwlock__(rwmtx); +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_rwlock__(rwmtx); +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rwunlock__(rwmtx); +} + +#endif /* pthread */ + + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#ifdef ETHR_MTX_HARD_DEBUG_Q +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int res; + long flgs = ethr_atomic_read(&mtxb->flgs); + + ETHR_MTX_HARD_ASSERT(res == 0); + + ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx); + + if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) { + ETHR_MTX_HARD_ASSERT(!mtxb->q); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(!rwmtx->rs); + } + } + else { + ethr_ts_event *tse; + int ws = 0, rs = 0, rsf = 0, ref = 0; + + ETHR_MTX_HARD_ASSERT(mtxb->q); + + tse = mtxb->q; + + do { + long type; + + ETHR_MTX_HARD_ASSERT(tse->next->prev == tse); + ETHR_MTX_HARD_ASSERT(tse->prev->next == tse); + + type = ethr_atomic_read(&tse->uaflgs); + ETHR_MTX_HARD_ASSERT(type == tse->uflgs); + switch (type) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + if (!rsf) + rsf = 1; + ETHR_MTX_HARD_ASSERT(!ref); + if (rwmtx->rq_end == tse) { + ETHR_MTX_HARD_ASSERT( + tse->next == rwmtx->mtxb.q + || tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ref = 1; + } + rs++; + break; + } + default: + ETHR_MTX_HARD_ASSERT(! "invalid wait type found"); + } + + tse = tse->next; + } while (tse != mtxb->q); + + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs); + } + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + ETHR_MTX_HARD_ASSERT(ws == mtxb->ws); +#endif + + if (flgs & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_MTX_HARD_ASSERT(ws); + else + ETHR_MTX_HARD_ASSERT(!ws); + + if (flgs & ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + ETHR_MTX_HARD_ASSERT(rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(rsf); + ETHR_MTX_HARD_ASSERT(ref); + ETHR_MTX_HARD_ASSERT(rs); + } + else { + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + } + ETHR_MTX_HARD_ASSERT(!rsf); + ETHR_MTX_HARD_ASSERT(!ref); + ETHR_MTX_HARD_ASSERT(!rs); + } + } +} + +#elif defined(ETHR_MTX_HARD_DEBUG_WSQ) + +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int ws = 0; + int rs = 0; + + if (mtxb->q) { + ethr_ts_event *tse = mtxb->q; + do { + switch (tse->uflgs) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: + rs++; + break; + default: + ETHR_MTX_HARD_ASSERT(0); + break; + } + tse = tse->next; + } while (tse != mtxb->q); + } + + ETHR_MTX_HARD_ASSERT(mtxb->ws == ws); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs); + } +} + +#endif + +#endif diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c deleted file mode 100644 index 9c88233934..0000000000 --- a/erts/lib_src/common/ethread.c +++ /dev/null @@ -1,3369 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* - * Description: A Thread library for use in the ERTS and other OTP - * applications. - * Author: Rickard Green - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#undef ETHR_STACK_GUARD_SIZE - -#if defined(ETHR_PTHREADS) - -#ifdef ETHR_TIME_WITH_SYS_TIME -# include <time.h> -# include <sys/time.h> -#else -# ifdef ETHR_HAVE_SYS_TIME_H -# include <sys/time.h> -# else -# include <time.h> -# endif -#endif -#include <sys/types.h> -#include <unistd.h> -#include <signal.h> - -#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE -# define ETHR_STACK_GUARD_SIZE (pagesize) -#endif - -#elif defined(ETHR_WIN32_THREADS) - -#undef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#include <process.h> -#include <winerror.h> - -#else -#error "Missing thread implementation" -#endif - -#include <limits.h> - -#define ETHR_FORCE_INLINE_FUNCS -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ -#include "ethread.h" - -#ifndef ETHR_HAVE_ETHREAD_DEFINES -#error Missing configure defines -#endif - -/* - * ---------------------------------------------------------------------------- - * Common stuff - * ---------------------------------------------------------------------------- - */ - -#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */ - -static int ethr_not_inited = 1; - -#define ASSERT(A) ETHR_ASSERT((A)) - -static void *(*allocp)(size_t) = malloc; -static void *(*reallocp)(void *, size_t) = realloc; -static void (*freep)(void *) = free; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; -#endif - -void *(*thread_create_prepare_func)(void) = NULL; -void (*thread_create_parent_func)(void *) = NULL; -void (*thread_create_child_func)(void *) = NULL; - -typedef struct ethr_xhndl_list_ ethr_xhndl_list; -struct ethr_xhndl_list_ { - ethr_xhndl_list *next; - void (*funcp)(void); -}; - -static size_t pagesize; -#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize) -static size_t min_stack_size; /* kilo words */ -static size_t max_stack_size; /* kilo words */ -#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) -#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) - -ethr_mutex xhndl_mtx; -ethr_xhndl_list *xhndl_list; - -static int -init_common(ethr_init_data *id) -{ - int res; - if (id) { - allocp = id->alloc; - reallocp = id->realloc; - freep = id->free; - thread_create_prepare_func = id->thread_create_prepare_func; - thread_create_parent_func = id->thread_create_parent_func; - thread_create_child_func = id->thread_create_child_func; - } - if (!allocp || !reallocp || !freep) - return EINVAL; - -#ifdef _SC_PAGESIZE - pagesize = (size_t) sysconf(_SC_PAGESIZE); -#elif defined(HAVE_GETPAGESIZE) - pagesize = (size_t) getpagesize(); -#else - pagesize = (size_t) 4*1024; /* Guess 4 KB */ -#endif - - /* User needs at least 4 KB */ - min_stack_size = 4*1024; -#if SIZEOF_VOID_P == 8 - /* Double that on 64-bit archs */ - min_stack_size *= 2; -#endif - /* On some systems as much as about 4 KB is used by the system */ - min_stack_size += 4*1024; - /* There should be room for signal handlers */ -#ifdef SIGSTKSZ - min_stack_size += SIGSTKSZ; -#else - min_stack_size += pagesize; -#endif - /* The system may think that we need more stack */ -#if defined(PTHREAD_STACK_MIN) - if (min_stack_size < PTHREAD_STACK_MIN) - min_stack_size = PTHREAD_STACK_MIN; -#elif defined(_SC_THREAD_STACK_MIN) - { - size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); - if (min_stack_size < thr_min_stk_sz) - min_stack_size = thr_min_stk_sz; - } -#endif - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ -#ifdef ETHR_STACK_GUARD_SIZE - min_stack_size += ETHR_STACK_GUARD_SIZE; -#endif - min_stack_size = ETHR_PAGE_ALIGN(min_stack_size); - - min_stack_size = ETHR_B2KW(min_stack_size); - - max_stack_size = 32*1024*1024; -#if SIZEOF_VOID_P == 8 - max_stack_size *= 2; -#endif - max_stack_size = ETHR_B2KW(max_stack_size); - - xhndl_list = NULL; - - res = ethr_mutex_init(&xhndl_mtx); - if (res != 0) - return res; - - res = ethr_mutex_set_forksafe(&xhndl_mtx); - if (res != 0 && res != ENOTSUP) - return res; - - return 0; -} - -int -ethr_install_exit_handler(void (*funcp)(void)) -{ - ethr_xhndl_list *xhp; - int res; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!funcp) - return EINVAL; - - xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list)); - if (!xhp) - return ENOMEM; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) { - (*freep)((void *) xhp); - return res; - } - - xhp->funcp = funcp; - xhp->next = xhndl_list; - xhndl_list = xhp; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - return res; -} - -static void -run_exit_handlers(void) -{ - int res; - ethr_xhndl_list *xhp; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) - abort(); - - xhp = xhndl_list; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - for (; xhp; xhp = xhp->next) - (*xhp->funcp)(); -} - -#if defined(ETHR_PTHREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * pthread implementation * -\* */ - -typedef struct { - pthread_mutex_t mtx; - pthread_cond_t cnd; - int initialized; - void *(*thr_func)(void *); - void *arg; - void *prep_func_res; -} thr_wrap_data_; - -static int no_ethreads; -static ethr_mutex no_ethrs_mtx; - -#ifndef ETHR_HAVE_PTHREAD_ATFORK -#define ETHR_HAVE_PTHREAD_ATFORK 0 -#endif - -#if !ETHR_HAVE_PTHREAD_ATFORK -#warning "Cannot enforce fork-safety" -#endif - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT -static pthread_rwlockattr_t write_pref_attr_data; -static pthread_rwlockattr_t *write_pref_attr; -#endif - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -/* - * Functions with safe_ prefix aborts on failure. To be used when - * we cannot recover after failure. - */ - -static ETHR_INLINE void -safe_mutex_lock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_lock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_mutex_unlock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_unlock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_cond_signal(pthread_cond_t *cndp) -{ - int res = pthread_cond_signal(cndp); - if (res != 0) - abort(); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static volatile int rec_mtx_attr_need_init = 1; -static pthread_mutexattr_t rec_mtx_attr; - -static int init_rec_mtx_attr(void); - -#endif - -#if ETHR_HAVE_PTHREAD_ATFORK - -static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER; - -static void lock_mutexes(void) -{ - ethr_mutex *m = &forksafe_mtx; - do { - - safe_mutex_lock(&m->pt_mtx); - - m = m->next; - - } while (m != &forksafe_mtx); -} - -static void unlock_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - - safe_mutex_unlock(&m->pt_mtx); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - -static void reinit_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - pthread_mutexattr_t *attrp = NULL; - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - if (m->is_rec_mtx) { - if (rec_mtx_attr_need_init) { - int res = init_rec_mtx_attr(); - if (res != 0) - abort(); - } - attrp = &rec_mtx_attr; - } -#endif - if (pthread_mutex_init(&m->pt_mtx, attrp) != 0) - abort(); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#endif - -static int -init_forksafe(void) -{ - static int init_done = 0; - int res = 0; - - if (init_done) - return res; - - forksafe_mtx.prev = &forksafe_mtx; - forksafe_mtx.next = &forksafe_mtx; - - res = pthread_atfork(lock_mutexes, - unlock_mutexes, -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - reinit_mutexes -#else - unlock_mutexes -#endif - ); - - init_done = 1; - return res; -} - -#endif - - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE); - -#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP); - -#else - -#error "Don't know how to set recursive mutex attributes" - -#endif - -static int -init_rec_mtx_attr(void) -{ - int res, mres; - static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER; - - mres = pthread_mutex_lock(&attrinit_mtx); - if (mres != 0) - return mres; - /* Got here under race conditions; check again ... */ - if (!rec_mtx_attr_need_init) - res = 0; - else { - res = pthread_mutexattr_init(&rec_mtx_attr); - if (res == 0) { - res = SET_REC_MUTEX_ATTR(&rec_mtx_attr); - if (res == 0) - rec_mtx_attr_need_init = 0; - else - (void) pthread_mutexattr_destroy(&rec_mtx_attr); - } - } - - mres = pthread_mutex_unlock(&attrinit_mtx); - if (mres != 0) - return mres; - return res; -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -static ETHR_INLINE void thr_exit_cleanup(void) -{ - run_exit_handlers(); - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); -} - -static void *thr_wrapper(void *vtwd) -{ - void *res; - thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd; - void *(*thr_func)(void *) = twd->thr_func; - void *arg = twd->arg; - - safe_mutex_lock(&twd->mtx); - - if (thread_create_child_func) - (*thread_create_child_func)(twd->prep_func_res); - - twd->initialized = 1; - - safe_cond_signal(&twd->cnd); - safe_mutex_unlock(&twd->mtx); - - res = (*thr_func)(arg); - thr_exit_cleanup(); - return res; -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ - int res; - - if (!ethr_not_inited) - return EINVAL; - - ethr_not_inited = 0; - - res = init_common(id); - if (res != 0) - goto error; - -#if ETHR_HAVE_PTHREAD_ATFORK - init_forksafe(); -#endif - - no_ethreads = 1; - res = ethr_mutex_init(&no_ethrs_mtx); - if (res != 0) - goto error; - res = ethr_mutex_set_forksafe(&no_ethrs_mtx); - if (res != 0 && res != ENOTSUP) - goto error; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { -#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK - res = pthread_spin_init(ðr_atomic_protection__[i].u.spnlck, 0); -#else - res = ethr_mutex_init(ðr_atomic_protection__[i].u.mtx); -#endif - if (res != 0) - goto error; - } - } -#endif - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT -#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \ - && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) - res = pthread_rwlockattr_init(&write_pref_attr_data); - if (res != 0) - goto error; - res = pthread_rwlockattr_setkind_np( - &write_pref_attr_data, - PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - if (res != 0) - goto error; - write_pref_attr = &write_pref_attr_data; -#else - write_pref_attr = NULL; -#endif -#endif - - - return 0; - - error: - ethr_not_inited = 1; - return res; - -} - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - thr_wrap_data_ twd; - pthread_attr_t attr; - int res, dres; - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - - twd.initialized = 0; - twd.thr_func = func; - twd.arg = arg; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Set som thread attributes */ - res = pthread_attr_init(&attr); - if (res != 0) - goto cleanup_parent_func; - res = pthread_mutex_init(&twd.mtx, NULL); - if (res != 0) - goto cleanup_attr_destroy; - res = pthread_cond_init(&twd.cnd, NULL); - if (res != 0) - goto cleanup_mutex_destroy; - - /* Schedule child thread in system scope (if possible) ... */ - res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); - if (res != 0 && res != ENOTSUP) - goto cleanup_cond_destroy; - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; - size_t stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif -#ifdef ETHR_STACK_GUARD_SIZE - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ - suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); -#endif - if (suggested_stack_size < min_stack_size) - stack_size = ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = ETHR_KW2B(max_stack_size); - else - stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - (void) pthread_attr_setstacksize(&attr, stack_size); - } - -#ifdef ETHR_STACK_GUARD_SIZE - (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); -#endif - - /* Detached or joinable... */ - res = pthread_attr_setdetachstate(&attr, - (opts && opts->detached - ? PTHREAD_CREATE_DETACHED - : PTHREAD_CREATE_JOINABLE)); - if (res != 0) - goto cleanup_cond_destroy; - - res = pthread_mutex_lock(&twd.mtx); - - if (res != 0) - goto cleanup_cond_destroy; - - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - if (no_ethreads < ETHR_MAX_THREADS) { - no_ethreads++; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - res = EAGAIN; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - goto cleanup_mutex_unlock; - } - - res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd); - - if (res != 0) { - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - - /* Wait for child to initialize... */ - while (!twd.initialized) { - res = pthread_cond_wait(&twd.cnd, &twd.mtx); - if (res != 0 && res != EINTR) - break; - } - - } - - /* Cleanup... */ - cleanup_mutex_unlock: - dres = pthread_mutex_unlock(&twd.mtx); - if (res == 0) - res = dres; - cleanup_cond_destroy: - dres = pthread_cond_destroy(&twd.cnd); - if (res == 0) - res = dres; - cleanup_mutex_destroy: - dres = pthread_mutex_destroy(&twd.mtx); - if (res == 0) - res = dres; - cleanup_attr_destroy: - dres = pthread_attr_destroy(&attr); - if (res == 0) - res = dres; - cleanup_parent_func: - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - return res; -} - -int -ethr_thr_join(ethr_tid tid, void **res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_join((pthread_t) tid, res); -} - -int -ethr_thr_detach(ethr_tid tid) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_detach((pthread_t) tid); -} - -void -ethr_thr_exit(void *res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - thr_exit_cleanup(); - pthread_exit(res); -} - -ethr_tid -ethr_self(void) -{ - return (ethr_tid) pthread_self(); -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - return pthread_equal((pthread_t) tid1, (pthread_t) tid2); -} - - -/* - * Mutex functions - */ - - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 0; - return pthread_mutex_init(&mtx->pt_mtx, NULL); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - if (rec_mtx_attr_need_init) - init_rec_mtx_attr(); - - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 1; - return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr); -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - if (mtx->next) { - ASSERT(mtx->prev); - ethr_mutex_unset_forksafe(mtx); - } -#if ETHR_XCHK - mtx->initialized = 0; -#endif - return pthread_mutex_destroy(&mtx->pt_mtx); -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (mtx->next) { - /* forksafe already set for this mutex */ - ASSERT(mtx->prev); - } - else { - mtx->next = forksafe_mtx.next; - mtx->prev = &forksafe_mtx; - forksafe_mtx.next->prev = mtx; - forksafe_mtx.next = mtx; - } - - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (!mtx->next) { - /* forksafe already unset for this mutex */ - ASSERT(!mtx->prev); - } - else { - mtx->prev->next = mtx->next; - mtx->next->prev = mtx->prev; - mtx->next = NULL; - mtx->prev = NULL; - } - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = ETHR_COND_INITIALIZED; -#endif - return pthread_cond_init(&cnd->pt_cnd, NULL); -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = 0; -#endif - return pthread_cond_destroy(&cnd->pt_cnd); -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_signal(&cnd->pt_cnd); -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_broadcast(&cnd->pt_cnd); -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - struct timespec to; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !timeout) { - ASSERT(0); - return EINVAL; - } -#endif - - to.tv_sec = timeout->tv_sec; - to.tv_nsec = timeout->tv_nsec; - - return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to); -} - - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr); -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); -#if ETHR_XCHK - rwmtx->initialized = 0; -#endif - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrlock__(rwmtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rlock__(rwmtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_runlock__(rwmtx); -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrwlock__(rwmtx); -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwlock__(rwmtx); -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwunlock__(rwmtx); -} - -#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -/* - * Current time - */ - -int -ethr_time_now(ethr_timeval *time) -{ - int res; - struct timeval tv; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - - res = gettimeofday(&tv, NULL); - time->tv_sec = (long) tv.tv_sec; - time->tv_nsec = ((long) tv.tv_usec)*1000; - return res; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_key_create((pthread_key_t *) keyp, NULL); -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_key_delete((pthread_key_t) key); -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_setspecific((pthread_key_t) key, value); -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return pthread_getspecific((pthread_key_t) key); -} - -/* - * Signal functions - */ - -#if ETHR_HAVE_ETHR_SIG_FUNCS - -int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set && !oset) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_sigmask(how, set, oset); -} - -int ethr_sigwait(const sigset_t *set, int *sig) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set || !sig) { - ASSERT(0); - return EINVAL; - } -#endif - if (sigwait(set, sig) < 0) - return errno; - return 0; -} - -#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ - -#elif defined(ETHR_WIN32_THREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Native win32 threads implementation * -\* */ - -#define INVALID_TID -1 - -/* The spin count values are more or less taken out of the blue */ -#define ETHR_MUTEX_SPIN_COUNT 5000 -#define ETHR_COND_SPIN_COUNT 1000 - -ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */ -ethr_tid last_serial; /* Last thread table serial used */ -ethr_tid last_ix; /* Last thread table index used */ -ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */ - -/* Event used for conditional variables. On per thread. */ -/*typedef struct cnd_wait_event__ cnd_wait_event_;*/ -struct cnd_wait_event__ { - HANDLE handle; - cnd_wait_event_ *prev; - cnd_wait_event_ *next; - int in_queue; -}; - -/* Thread specific data. Stored in the thread table */ -typedef struct { - ethr_tid thr_id; - HANDLE thr_handle; - ethr_tid joiner; - void *result; - cnd_wait_event_ wait_event; -} thr_data_; - -/* Argument passed to thr_wrapper() */ -typedef struct { - void * (*func)(void *); - void * arg; - thr_data_ *ptd; - thr_data_ *td; - int res; - void *prep_func_res; -} thr_wrap_data_; - - -static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect - the thread table from concurrent - accesses. */ -static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect - initialazition of 'statically - initialized' mutexes */ -static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */ - -static DWORD tls_own_thr_data; - -static thr_data_ main_thr_data; - -#define THR_IX(TID) ((TID) & thr_ix_mask) -#define OWN_THR_DATA ((thr_data_ *) TlsGetValue(tls_own_thr_data)) - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -static int -get_errno(void) -{ - switch (GetLastError()) { - case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ - case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ - case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ - case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ - case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ - case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ - case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ - case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ - case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ - case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ - case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ - case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ - case ERROR_INVALID_DATA: return EINVAL; /* 13 */ - case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ - case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ - case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ - case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ - case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ - case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ - case ERROR_BAD_UNIT: return EACCES; /* 20 */ - case ERROR_NOT_READY: return EACCES; /* 21 */ - case ERROR_BAD_COMMAND: return EACCES; /* 22 */ - case ERROR_CRC: return EACCES; /* 23 */ - case ERROR_BAD_LENGTH: return EACCES; /* 24 */ - case ERROR_SEEK: return EACCES; /* 25 */ - case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ - case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ - case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ - case ERROR_WRITE_FAULT: return EACCES; /* 29 */ - case ERROR_READ_FAULT: return EACCES; /* 30 */ - case ERROR_GEN_FAILURE: return EACCES; /* 31 */ - case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ - case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ - case ERROR_WRONG_DISK: return EACCES; /* 34 */ - case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ - case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ - case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ - case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ - case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ - case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ - case ERROR_FAIL_I24: return EACCES; /* 83 */ - case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ - case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ - case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ - case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ - case ERROR_DISK_FULL: return ENOSPC; /* 112 */ - case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ - case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ - case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ - case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ - case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ - case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ - case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ - case ERROR_NOT_LOCKED: return EACCES; /* 158 */ - case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ - case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ - case ERROR_LOCK_FAILED: return EACCES; /* 167 */ - case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ - case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ - case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ - case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ - case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ - case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ - case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ - case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ - case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ - case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ - case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ - case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ - case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ - case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ - case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ - case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ - case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ - case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ - case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ - default: return EINVAL; - } -} - -static ETHR_INLINE thr_data_ * -tid2thr(ethr_tid tid) -{ - ethr_tid ix; - thr_data_ *td; - - if (tid < 0) - return NULL; - ix = THR_IX(tid); - if (ix >= ETHR_MAX_THREADS) - return NULL; - td = thr_table[ix]; - if (!td) - return NULL; - if (td->thr_id != tid) - return NULL; - return td; -} - -static ETHR_INLINE void -new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix) -{ - ethr_tid tmp_serial = last_serial; - ethr_tid tmp_ix = last_ix + 1; - ethr_tid start_ix = tmp_ix; - - - do { - if (tmp_ix >= ETHR_MAX_THREADS) { - tmp_serial++; - if ((tmp_serial << serial_shift) < 0) - tmp_serial = 0; - tmp_ix = 0; - } - if (!thr_table[tmp_ix]) { - *new_tid = (tmp_serial << serial_shift) | tmp_ix; - *new_serial = tmp_serial; - *new_ix = tmp_ix; - return; - } - tmp_ix++; - } while (tmp_ix != start_ix); - - *new_tid = INVALID_TID; - *new_serial = INVALID_TID; - *new_ix = INVALID_TID; - -} - - -static void thr_exit_cleanup(thr_data_ *td, void *res) -{ - - ASSERT(td == OWN_THR_DATA); - - run_exit_handlers(); - - EnterCriticalSection(&thr_table_cs); - CloseHandle(td->wait_event.handle); - if (td->thr_handle == INVALID_HANDLE_VALUE) { - /* We are detached; cleanup thread table */ - ASSERT(td->joiner == INVALID_TID); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - } - else { - /* Save result and let joining thread cleanup */ - td->result = res; - } - LeaveCriticalSection(&thr_table_cs); -} - -static unsigned __stdcall thr_wrapper(LPVOID args) -{ - void *(*func)(void*) = ((thr_wrap_data_ *) args)->func; - void *arg = ((thr_wrap_data_ *) args)->arg; - thr_data_ *td = ((thr_wrap_data_ *) args)->td; - - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE - || !TlsSetValue(tls_own_thr_data, (LPVOID) td)) { - ((thr_wrap_data_ *) args)->res = get_errno(); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - _endthreadex((unsigned) 0); - ASSERT(0); - } - - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - - if (thread_create_child_func) - (*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res); - - ASSERT(td == OWN_THR_DATA); - - ((thr_wrap_data_ *) args)->res = 0; - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - - thr_exit_cleanup(td, (*func)(arg)); - return 0; -} - -int -ethr_fake_static_mutex_init(ethr_mutex *mtx) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!mtx->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, - ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -static int -fake_static_cond_init(ethr_cond *cnd) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!cnd->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, - ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -#ifdef __GNUC__ -#define LL_LITERAL(X) X##LL -#else -#define LL_LITERAL(X) X##i64 -#endif - -#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) - -static ETHR_INLINE void -get_curr_time(long *sec, long *nsec) -{ - SYSTEMTIME t; - FILETIME ft; - LONGLONG lft; - - GetSystemTime(&t); - SystemTimeToFileTime(&t, &ft); - memcpy(&lft, &ft, sizeof(lft)); - *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; - *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); -} - -static cnd_wait_event_ *cwe_freelist; -static CRITICAL_SECTION cwe_cs; - -static int -alloc_cwe(cnd_wait_event_ **cwe_res) -{ - cnd_wait_event_ *cwe; - EnterCriticalSection(&cwe_cs); - cwe = cwe_freelist; - if (cwe) { - cwe_freelist = cwe->next; - LeaveCriticalSection(&cwe_cs); - } - else { - LeaveCriticalSection(&cwe_cs); - cwe = (*allocp)(sizeof(cnd_wait_event_)); - if (!cwe) - return ENOMEM; - cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (cwe->handle == INVALID_HANDLE_VALUE) { - int res = get_errno(); - (*freep)(cwe); - return res; - } - } - *cwe_res = cwe; - return 0; -} - -static -free_cwe(cnd_wait_event_ *cwe) -{ - EnterCriticalSection(&cwe_cs); - cwe->next = cwe_freelist; - cwe_freelist = cwe; - LeaveCriticalSection(&cwe_cs); -} - -static ETHR_INLINE int -condwait(ethr_cond *cnd, - ethr_mutex *mtx, - int with_timeout, - ethr_timeval *timeout) -{ - int res; - thr_data_ *td; - cnd_wait_event_ *cwe; - DWORD code; - long time; /* time until timeout in milli seconds */ - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - - if (!mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || (with_timeout && !timeout)) { - ASSERT(0); - return EINVAL; - } -#endif - - td = OWN_THR_DATA; - if (td) - cwe = &td->wait_event; - else { /* A non-ethread thread */ - res = alloc_cwe(&cwe); - if (res != 0) - return res; - } - - if (!cnd->initialized) - fake_static_cond_init(cnd); - EnterCriticalSection(&cnd->cs); - - ASSERT(!cwe->in_queue); - if (cnd->queue_end) { - ASSERT(cnd->queue); - cwe->prev = cnd->queue_end; - cwe->next = NULL; - cnd->queue_end->next = cwe; - cnd->queue_end = cwe; - } - else { - ASSERT(!cnd->queue); - cwe->prev = NULL; - cwe->next = NULL; - cnd->queue = cwe; - cnd->queue_end = cwe; - } - cwe->in_queue = 1; - - LeaveCriticalSection(&cnd->cs); - - LeaveCriticalSection(&mtx->cs); - - if (!with_timeout) - time = INFINITE; - else { - long sec, nsec; - ASSERT(timeout); - get_curr_time(&sec, &nsec); - time = (timeout->tv_sec - sec)*1000; - time += (timeout->tv_nsec - nsec + 500)/1000000; - if (time < 0) - time = 0; - } - - /* wait for event to signal */ - code = WaitForSingleObject(cwe->handle, time); - - EnterCriticalSection(&mtx->cs); - - if (code == WAIT_OBJECT_0) { - /* We were woken by a signal or a broadcast ... */ - res = 0; - - /* ... no need to remove event from wait queue since this was - taken care of by the signal or broadcast */ -#ifdef DEBUG - EnterCriticalSection(&cnd->cs); - ASSERT(!cwe->in_queue); - LeaveCriticalSection(&cnd->cs); -#endif - - } - else { - /* We timed out... */ - res = ETIMEDOUT; - - /* ... probably have to remove event from wait queue ... */ - EnterCriticalSection(&cnd->cs); - - if (cwe->in_queue) { /* ... but we must check that we are in queue - since a signal or broadcast after timeout - may have removed us from the queue */ - if (cwe->prev) { - cwe->prev->next = cwe->next; - } - else { - ASSERT(cnd->queue == cwe); - cnd->queue = cwe->next; - } - - if (cwe->next) { - cwe->next->prev = cwe->prev; - } - else { - ASSERT(cnd->queue_end == cwe); - cnd->queue_end = cwe->prev; - } - cwe->in_queue = 0; - } - - LeaveCriticalSection(&cnd->cs); - - } - - if (!td) - free_cwe(cwe); - - return res; - -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ -#ifdef _WIN32_WINNT - DWORD major = (_WIN32_WINNT >> 8) & 0xff; - DWORD minor = _WIN32_WINNT & 0xff; - OSVERSIONINFO os_version; -#endif - int err = 0; - thr_data_ *td = &main_thr_data; - unsigned long i; - - if (!ethr_not_inited) - return EINVAL; - -#ifdef _WIN32_WINNT - os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - GetVersionEx(&os_version); - if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT - || os_version.dwMajorVersion < major - || (os_version.dwMajorVersion == major - && os_version.dwMinorVersion < minor)) - return ENOTSUP; -#endif - - ASSERT(ETHR_MAX_THREADS > 0); - for (i = ETHR_MAX_THREADS - 1, serial_shift = 0; - i; - serial_shift++, i >>= 1); - thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift); - - tls_own_thr_data = TlsAlloc(); - if (tls_own_thr_data == TLS_OUT_OF_INDEXES) - goto error; - - last_serial = 0; - last_ix = 0; - - td->thr_id = 0; - td->thr_handle = GetCurrentThread(); - td->joiner = INVALID_TID; - td->result = NULL; - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE) - goto error; - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - thr_table[0] = td; - - if (!TlsSetValue(tls_own_thr_data, (LPVOID) td)) - goto error; - - ASSERT(td == OWN_THR_DATA); - - - cwe_freelist = NULL; - if (!InitializeCriticalSectionAndSpinCount(&cwe_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - - for (i = 1; i < ETHR_MAX_THREADS; i++) - thr_table[i] = NULL; - - if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - ethr_not_inited = 0; - - err = init_common(id); - if (err) - goto error; - - return 0; - - error: - ethr_not_inited = 1; - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - if (td->thr_handle != INVALID_HANDLE_VALUE) - CloseHandle(td->thr_handle); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - return err; -} - -/* - * Thread functions. - */ - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - int err = 0; - thr_wrap_data_ twd; - thr_data_ *my_td, *child_td = NULL; - ethr_tid child_tid, child_serial, child_ix; - DWORD code; - unsigned ID; - unsigned stack_size = 0; /* 0 = system default */ - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - my_td = OWN_THR_DATA; - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif - if (suggested_stack_size < min_stack_size) - stack_size = (unsigned) ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = (unsigned) ETHR_KW2B(max_stack_size); - else - stack_size = - (unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - } - - EnterCriticalSection(&thr_table_cs); - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Find a new thread id to use */ - new_tid(&child_tid, &child_serial, &child_ix); - if (child_tid == INVALID_TID) { - err = EAGAIN; - goto error; - } - - ASSERT(child_ix == THR_IX(child_tid)); - - *tid = child_tid; - - ASSERT(!thr_table[child_ix]); - - /* Alloc thread data */ - thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_)); - if (!child_td) { - err = ENOMEM; - goto error; - } - - /* Init thread data */ - - child_td->thr_id = child_tid; - child_td->thr_handle = INVALID_HANDLE_VALUE; - child_td->joiner = INVALID_TID; - child_td->result = NULL; - /* 'child_td->wait_event' is initialized by child thread */ - - - /* Init thread wrapper data */ - - twd.func = func; - twd.arg = arg; - twd.ptd = my_td; - twd.td = child_td; - twd.res = 0; - - ASSERT(!my_td->wait_event.in_queue); - - /* spawn the thr_wrapper function */ - child_td->thr_handle = (HANDLE) _beginthreadex(NULL, - stack_size, - thr_wrapper, - (LPVOID) &twd, - 0, - &ID); - if (child_td->thr_handle == (HANDLE) 0) { - child_td->thr_handle = INVALID_HANDLE_VALUE; - goto error; - } - - ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE); - - /* Wait for child to finish initialization */ - code = WaitForSingleObject(my_td->wait_event.handle, INFINITE); - if (twd.res || code != WAIT_OBJECT_0) { - err = twd.res; - goto error; - } - - if (opts && opts->detached) { - CloseHandle(child_td->thr_handle); - child_td->thr_handle = INVALID_HANDLE_VALUE; - } - - last_serial = child_serial; - last_ix = child_ix; - - ASSERT(thr_table[child_ix] == child_td); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - if (child_ix != INVALID_TID) { - - if (child_td) { - ASSERT(thr_table[child_ix] == child_td); - - if (child_td->thr_handle != INVALID_HANDLE_VALUE) { - WaitForSingleObject(child_td->thr_handle, INFINITE); - CloseHandle(child_td->thr_handle); - } - - (*freep)((void *) child_td); - thr_table[child_ix] = NULL; - } - } - - *tid = INVALID_TID; - - LeaveCriticalSection(&thr_table_cs); - return err; -} - -int ethr_thr_join(ethr_tid tid, void **res) -{ - int err = 0; - DWORD code; - thr_data_ *td; - thr_data_ *my_td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - my_td = OWN_THR_DATA; - - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - err = ESRCH; - else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone else is joining */ - err = EINVAL; - else if (my_td == td) - err = EDEADLK; - else - td->joiner = my_td->thr_id; - - LeaveCriticalSection(&thr_table_cs); - - if (err) - goto error; - - /* Wait for thread to terminate */ - code = WaitForSingleObject(td->thr_handle, INFINITE); - if (code != WAIT_OBJECT_0) - goto error; - - EnterCriticalSection(&thr_table_cs); - - ASSERT(td == tid2thr(tid)); - ASSERT(td->thr_handle != INVALID_HANDLE_VALUE); - ASSERT(td->joiner == my_td->thr_id); - - if (res) - *res = td->result; - - CloseHandle(td->thr_handle); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - return err; -} - - -int -ethr_thr_detach(ethr_tid tid) -{ - int res; - DWORD code; - thr_data_ *td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!OWN_THR_DATA) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - res = ESRCH; - if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone is joining */ - res = EINVAL; - else { - res = 0; - CloseHandle(td->thr_handle); - td->thr_handle = INVALID_HANDLE_VALUE; - } - - LeaveCriticalSection(&thr_table_cs); - - return res; -} - - -void -ethr_thr_exit(void *res) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - td = OWN_THR_DATA; - if (!td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return; - } - thr_exit_cleanup(td, res); - _endthreadex((unsigned) 0); -} - -ethr_tid -ethr_self(void) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return INVALID_TID; - } -#endif - /* It is okay for non-ethreads (i.e. native win32 threads) to call - ethr_self(). They will however be returned the INVALID_TID. */ - td = OWN_THR_DATA; - if (!td) - return INVALID_TID; - return td->thr_id; -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - /* INVALID_TID does not equal any tid, not even the INVALID_TID */ - return tid1 == tid2 && tid1 != INVALID_TID; -} - -/* - * Mutex functions. - */ - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#if ETHR_XCHK - mtx->is_rec_mtx = 0; -#endif - return 0; -} - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ - int res; - res = ethr_mutex_init(mtx); -#if ETHR_XCHK - mtx->is_rec_mtx = 1; -#endif - return res; -} - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&mtx->cs); - mtx->initialized = 0; - return 0; -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - int res; - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions. - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - return 0; -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || cnd->queue) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&cnd->cs); - cnd->initialized = 0; - return 0; -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - cwe = cnd->queue; - if (cwe) { - ASSERT(cwe->in_queue); - SetEvent(cnd->queue->handle); - if (cwe->next) - cwe->next->prev = NULL; - else { - ASSERT(cnd->queue_end == cnd->queue); - cnd->queue_end = NULL; - } - cnd->queue = cwe->next; - cwe->in_queue = 0; - } - LeaveCriticalSection(&cnd->cs); - return 0; -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - for (cwe = cnd->queue; cwe; cwe = cwe->next) { - ASSERT(cwe->in_queue); - SetEvent(cwe->handle); - cwe->in_queue = 0; - } - cnd->queue = NULL; - cnd->queue_end = NULL; - LeaveCriticalSection(&cnd->cs); - return 0; - -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ - return condwait(cnd, mtx, 0, NULL); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - return condwait(cnd, mtx, 1, timeout); -} - -int -ethr_time_now(ethr_timeval *time) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - get_curr_time(&time->tv_sec, &time->tv_nsec); - return 0; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ - DWORD key; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - key = TlsAlloc(); - if (key == TLS_OUT_OF_INDEXES) - return get_errno(); - *keyp = (ethr_tsd_key) key; - return 0; -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsFree((DWORD) key)) - return get_errno(); - return 0; -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsSetValue((DWORD) key, (LPVOID) value)) - return get_errno(); - return 0; -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return (void *) TlsGetValue((DWORD) key); -} - -/* Misc */ - -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS - -int -ethr_do_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -int -ethr_do_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - lock->counter = 0; - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - -#else -#error "Missing thread implementation" -#endif - -/* Atomics */ - -int -ethr_atomic_init(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_init__(var, i); -} - -int -ethr_atomic_set(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_set__(var, i); -} - -int -ethr_atomic_read(ethr_atomic_t *var, long *i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !i) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_read__(var, i); -} - - -int -ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_addtest__(var, incr, testp); -} - -int -ethr_atomic_inctest(ethr_atomic_t *incp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inctest__(incp, testp); -} - -int -ethr_atomic_dectest(ethr_atomic_t *decp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dectest__(decp, testp); -} - -int -ethr_atomic_add(ethr_atomic_t *var, long incr) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_add__(var, incr); -} - -int -ethr_atomic_inc(ethr_atomic_t *incp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inc__(incp); -} - -int -ethr_atomic_dec(ethr_atomic_t *decp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dec__(decp); -} - -int -ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_and_old__(var, mask, old); -} - -int -ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_or_old__(var, mask, old); -} - -int -ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_xchg__(var, new, old); -} - -int -ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_cmpxchg__(var, new, expected, old); -} - -/* Spinlocks and rwspinlocks */ - -int -ethr_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_init__(lock); -} - -int -ethr_spinlock_destroy(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_destroy__(lock); -} - - -int -ethr_spin_unlock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_unlock__(lock); -} - -int -ethr_spin_lock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_lock__(lock); -} - -int -ethr_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_init__(lock); -} - -int -ethr_rwlock_destroy(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_destroy__(lock); -} - -int -ethr_read_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_unlock__(lock); -} - -int -ethr_read_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_lock__(lock); -} - -int -ethr_write_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_unlock__(lock); -} - -int -ethr_write_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_lock__(lock); -} - - -int -ethr_gate_init(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&gp->mtx); - if (res != 0) - return res; - res = ethr_cond_init(&gp->cnd); - if (res != 0) { - ethr_mutex_destroy(&gp->mtx); - return res; - } - gp->open = 0; - return 0; -} - -int -ethr_gate_destroy(ethr_gate *gp) -{ - int res, dres; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_destroy(&gp->mtx); - dres = ethr_cond_destroy(&gp->cnd); - if (res == 0) - res = dres; - gp->open = 0; - return res; -} - -int -ethr_gate_close(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open = 0; - res = ethr_mutex_unlock__(&gp->mtx); - return res; -} - -int -ethr_gate_let_through(ethr_gate *gp, unsigned no) -{ - int res, ures; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open += no; - res = (gp->open == 1 - ? ethr_cond_signal(&gp->cnd) - : ethr_cond_broadcast(&gp->cnd)); - ures = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - res = ures; - return res; -} - -int -ethr_gate_swait(ethr_gate *gp, int spincount) -{ - int res, ures, n; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - n = spincount; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - while (n >= 0 && !gp->open) { - res = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - return res; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - n--; - } - while (!gp->open) { - res = ethr_cond_wait(&gp->cnd, &gp->mtx); - if (res != 0 && res != EINTR) - goto done; - } - gp->open--; - done: - ures = ethr_mutex_unlock__(&gp->mtx); - if (res == 0) - res = ures; - return res; -} - - -int -ethr_gate_wait(ethr_gate *gp) -{ - return ethr_gate_swait(gp, 0); -} - - -/* rwmutex fallback */ -#ifdef ETHR_USE_RWMTX_FALLBACK - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&rwmtx->mtx); - if (res != 0) - return res; - ethr_cond_init(&rwmtx->rcnd); - if (res != 0) - goto error_cleanup1; - res = ethr_cond_init(&rwmtx->wcnd); - if (res != 0) - goto error_cleanup2; - rwmtx->readers = 0; - rwmtx->waiting_readers = 0; - rwmtx->waiting_writers = 0; -#if ETHR_XCHK - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return 0; - error_cleanup2: - ethr_cond_destroy(&rwmtx->rcnd); - error_cleanup1: - ethr_mutex_destroy(&rwmtx->mtx); - return res; -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res, pres; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = 0; -#endif - res = ethr_mutex_destroy(&rwmtx->mtx); - pres = ethr_cond_destroy(&rwmtx->rcnd); - if (res == 0) - res = pres; - pres = ethr_cond_destroy(&rwmtx->wcnd); - if (res == 0) - res = pres; - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (rwmtx->waiting_writers) { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - while (rwmtx->waiting_writers) { - rwmtx->waiting_readers++; - res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx); - rwmtx->waiting_readers--; - if (res != 0 && res != EINTR) { - (void) ethr_mutex_unlock__(&rwmtx->mtx); - return res; - } - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - rwmtx->readers--; - if (!rwmtx->readers && rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - else { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - - while (rwmtx->readers) { - rwmtx->waiting_writers++; - res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx); - rwmtx->waiting_writers--; - if (res != 0 && res != EINTR) { - (void) ethr_rwmutex_rwunlock(rwmtx); - return res; - } - } - return 0; -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = 0; - if (rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - else if (rwmtx->waiting_readers) - res = ethr_cond_broadcast(&rwmtx->rcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */ - -void -ethr_compiler_barrier(void) -{ - -} - -#ifdef DEBUG - -#include <stdio.h> -int ethr_assert_failed(char *f, int l, char *a) -{ - fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a); - abort(); - return 0; -} - -#endif - - diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c new file mode 100644 index 0000000000..6731c0eb46 --- /dev/null +++ b/erts/lib_src/pthread/ethr_event.c @@ -0,0 +1,219 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#include "ethread.h" + +#if defined(ETHR_LINUX_FUTEX_IMPL__) +/* --- Linux futex implementation of ethread events ------------------------- */ + +#include <sched.h> +#include <errno.h> + +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 + +int +ethr_event_init(ethr_event *e) +{ + ethr_atomic_init(&e->futex, ETHR_EVENT_OFF__); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + unsigned sc = spincount; + int res; + long val; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + while (1) { + val = ethr_atomic_read(&e->futex); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic_cmpxchg(&e->futex, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAIT__, ETHR_EVENT_OFF_WAITER__); + if (res == EINTR) + break; + if (res != 0 && res != EWOULDBLOCK) + ETHR_FATAL_ERROR__(res); + } + + return res; +} + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +int +ethr_event_init(ethr_event *e) +{ + int res; + ethr_atomic_init(&e->state, ETHR_EVENT_OFF__); + res = pthread_mutex_init(&e->mtx, NULL); + if (res != 0) + return res; + res = pthread_cond_init(&e->cnd, NULL); + if (res != 0) { + pthread_mutex_destroy(&e->mtx); + return res; + } + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + int res; + res = pthread_mutex_destroy(&e->mtx); + if (res != 0) + return res; + res = pthread_cond_destroy(&e->cnd); + if (res != 0) + return res; + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + int sc = spincount; + long val; + int res, ulres; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + val = ethr_atomic_read(&e->state); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__ + || val == ETHR_EVENT_OFF__); + + res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + + while (1) { + + val = ethr_atomic_read(&e->state); + if (val == ETHR_EVENT_ON__) + break; + + res = pthread_cond_wait(&e->cnd, &e->mtx); + if (res == EINTR) + break; + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + + ulres = pthread_mutex_unlock(&e->mtx); + if (ulres != 0) + ETHR_FATAL_ERROR__(ulres); + + return res; /* 0 || EINTR */ +} + +#else +#error No ethread event implementation +#endif + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait__(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait__(e, spincount); +} diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c new file mode 100644 index 0000000000..ea1d9d43f0 --- /dev/null +++ b/erts/lib_src/pthread/ethread.c @@ -0,0 +1,477 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Pthread implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#include <stdio.h> +#ifdef ETHR_TIME_WITH_SYS_TIME +# include <time.h> +# include <sys/time.h> +#else +# ifdef ETHR_HAVE_SYS_TIME_H +# include <sys/time.h> +# else +# include <time.h> +# endif +#endif +#include <sys/types.h> +#include <unistd.h> +#include <signal.h> + +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +pthread_key_t ethr_ts_event_key__; +static int child_wait_spin_count; + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(void) +{ + ethr_run_exit_handlers__(); +} + + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_atomic_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +static void *thr_wrapper(void *vtwd) +{ + long result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (long) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(); + return res; +} + +/* internal exports */ + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return pthread_setspecific(ethr_ts_event_key__, (void *) tsep); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return pthread_getspecific(ethr_ts_event_key__); +} + +/* + * -------------------------------------------------------------------------- + * Exported functions + * -------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ + int res; + + if (!ethr_not_inited__) + return EINVAL; + + ethr_not_inited__ = 0; + + res = ethr_init_common__(id); + if (res != 0) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + res = pthread_key_create(ðr_ts_event_key__, ethr_ts_event_destructor__); + + return 0; + error: + ethr_not_inited__ = 1; + return res; + +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + ethr_thr_wrap_data__ twd; + pthread_attr_t attr; + int res, dres; + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + ethr_atomic_init(&twd.result, -1); + twd.tse = ethr_get_ts_event(); + twd.thr_func = func; + twd.arg = arg; + + res = pthread_attr_init(&attr); + if (res != 0) + return res; + + /* Error cleanup needed after this point */ + + /* Schedule child thread in system scope (if possible) ... */ + res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); + if (res != 0 && res != ENOTSUP) + goto error; + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; + size_t stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif +#ifdef ETHR_STACK_GUARD_SIZE + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ + suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + (void) pthread_attr_setstacksize(&attr, stack_size); + } + +#ifdef ETHR_STACK_GUARD_SIZE + (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); +#endif + + /* Detached or joinable... */ + res = pthread_attr_setdetachstate(&attr, + (opts && opts->detached + ? PTHREAD_CREATE_DETACHED + : PTHREAD_CREATE_JOINABLE)); + if (res != 0) + goto error; + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd); + + if (res == 0) { + int spin_count = child_wait_spin_count; + + /* Wait for child to initialize... */ + while (1) { + long result; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + res = (int) result; + goto error; + } + + res = ethr_event_swait(&twd.tse->event, spin_count); + if (res != 0 && res != EINTR) + goto error; + spin_count = 0; + } + } + + /* Cleanup... */ + + error: + dres = pthread_attr_destroy(&attr); + if (res == 0) + res = dres; + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + return res; +} + +int +ethr_thr_join(ethr_tid tid, void **res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_join((pthread_t) tid, res); +} + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_detach((pthread_t) tid); +} + +void +ethr_thr_exit(void *res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + thr_exit_cleanup(); + pthread_exit(res); +} + +ethr_tid +ethr_self(void) +{ + return (ethr_tid) pthread_self(); +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + return pthread_equal((pthread_t) tid1, (pthread_t) tid2); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +/* + * Current time + */ + +int +ethr_time_now(ethr_timeval *time) +{ + int res; + struct timeval tv; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!time) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + res = gettimeofday(&tv, NULL); + time->tv_sec = (long) tv.tv_sec; + time->tv_nsec = ((long) tv.tv_usec)*1000; + return res; +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_key_create((pthread_key_t *) keyp, NULL); +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_key_delete((pthread_key_t) key); +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_setspecific((pthread_key_t) key, value); +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return pthread_getspecific((pthread_key_t) key); +} + +/* + * Signal functions + */ + +#if ETHR_HAVE_ETHR_SIG_FUNCS + +int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set && !oset) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_sigmask(how, set, oset); +} + +int ethr_sigwait(const sigset_t *set, int *sig) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set || !sig) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + if (sigwait(set, sig) < 0) + return errno; + return 0; +} + +#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ + abort(); +} diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c new file mode 100644 index 0000000000..ddb4780ff1 --- /dev/null +++ b/erts/lib_src/win/ethr_event.c @@ -0,0 +1,120 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#include "ethread.h" + +/* --- Windows implementation of thread events ------------------------------ */ + +int +ethr_event_init(ethr_event *e) +{ + e->state = ETHR_EVENT_OFF__; + e->handle = CreateEvent(NULL, FALSE, FALSE, NULL); + if (e->handle == INVALID_HANDLE_VALUE) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + BOOL res = CloseHandle(e->handle); + return res == 0 ? ethr_win_get_errno__() : 0; +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +static ETHR_INLINE int +wait(ethr_event *e, int spincount) +{ + LONG state; + DWORD code; + int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + sc = spincount; + + while (1) { + long on; + while (1) { +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + state = e->state; +#else + state = InterlockedExchangeAdd(&e->state, (LONG) 0); +#endif + if (state == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (state != ETHR_EVENT_OFF_WAITER__) { + state = _InterlockedCompareExchange(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (state == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(state == ETHR_EVENT_OFF__); + } + + code = WaitForSingleObject(e->handle, INFINITE); + if (code != WAIT_OBJECT_0) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } + +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait(e, spincount); +} diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c new file mode 100644 index 0000000000..69523edf94 --- /dev/null +++ b/erts/lib_src/win/ethread.c @@ -0,0 +1,625 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Windows native threads implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <winerror.h> +#include <stdio.h> +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_tid *tid; + ethr_atomic_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +#define ETHR_INVALID_TID_ID -1 + +struct ethr_join_data_ { + HANDLE handle; + void *res; +}; + +static ethr_atomic_t thread_id_counter; +static DWORD own_tid_key; +static ethr_tid main_thr_tid; +static int child_wait_spin_count; + +DWORD ethr_ts_event_key__; + +#define ETHR_GET_OWN_TID__ ((ethr_tid *) TlsGetValue(own_tid_key)) + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(ethr_tid *tid, void *res) +{ + + ETHR_ASSERT(tid == ETHR_GET_OWN_TID__); + + if (tid->jdata) + tid->jdata->res = res; + + ethr_run_exit_handlers__(); + ethr_ts_event_destructor__((void *) ethr_get_tse__()); +} + +static unsigned __stdcall thr_wrapper(LPVOID vtwd) +{ + ethr_tid my_tid; + long result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (long) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + my_tid = *twd->tid; + if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) { + result = (long) ethr_win_get_errno__(); + ethr_free_ts_event__(tsep); + } + else { + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(&my_tid, res); + return 0; +} + +#ifdef __GNUC__ +#define LL_LITERAL(X) X##LL +#else +#define LL_LITERAL(X) X##i64 +#endif + +#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) + +static ETHR_INLINE void +get_curr_time(long *sec, long *nsec) +{ + SYSTEMTIME t; + FILETIME ft; + LONGLONG lft; + + GetSystemTime(&t); + SystemTimeToFileTime(&t, &ft); + memcpy(&lft, &ft, sizeof(lft)); + *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; + *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); +} + +/* internal exports */ + +int +ethr_win_get_errno__(void) +{ + return erts_get_last_win_errno(); +} + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return (TlsSetValue(ethr_ts_event_key__, (LPVOID) tsep) + ? 0 + : ethr_win_get_errno__()); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__); +} + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ +#if 1 + DebugBreak(); +#else + abort(); +#endif +} + +/* + * ---------------------------------------------------------------------------- + * Exported functions + * ---------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ +#ifdef _WIN32_WINNT + DWORD major = (_WIN32_WINNT >> 8) & 0xff; + DWORD minor = _WIN32_WINNT & 0xff; + OSVERSIONINFO os_version; +#endif + int err = 0; + unsigned long i; + + if (!ethr_not_inited__) + return EINVAL; + +#ifdef _WIN32_WINNT + os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + GetVersionEx(&os_version); + if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT + || os_version.dwMajorVersion < major + || (os_version.dwMajorVersion == major + && os_version.dwMinorVersion < minor)) + return ENOTSUP; +#endif + err = ethr_init_common__(id); + if (err) + goto error; + + own_tid_key = TlsAlloc(); + if (own_tid_key == TLS_OUT_OF_INDEXES) + goto error; + + ethr_atomic_init(&thread_id_counter, 0); + + main_thr_tid.id = 0; + main_thr_tid.jdata = NULL; + + if (!TlsSetValue(own_tid_key, (LPVOID) &main_thr_tid)) + goto error; + + ETHR_ASSERT(&main_thr_tid == ETHR_GET_OWN_TID__); + + ethr_ts_event_key__ = TlsAlloc(); + if (ethr_ts_event_key__ == TLS_OUT_OF_INDEXES) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + ethr_not_inited__ = 0; + + return 0; + + error: + ethr_not_inited__ = 1; + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + return err; +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + + +/* + * Thread functions. + */ + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + HANDLE handle = INVALID_HANDLE_VALUE; + int err = 0; + ethr_thr_wrap_data__ twd; + DWORD code; + unsigned ID; + unsigned stack_size = 0; /* 0 = system default */ + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + do { + tid->id = ethr_atomic_inc_read(&thread_id_counter); + } while (tid->id == ETHR_INVALID_TID_ID); + + if (opts && opts->detached) + tid->jdata = NULL; + else { + tid->jdata = ethr_mem__.std.alloc(sizeof(struct ethr_join_data_)); + if (!tid->jdata) + return ENOMEM; + tid->jdata->handle = INVALID_HANDLE_VALUE; + tid->jdata->res = NULL; + } + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = (unsigned) + ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + } + + ethr_atomic_init(&twd.result, -1); + + twd.tid = tid; + twd.thr_func = func; + twd.arg = arg; + twd.tse = ethr_get_ts_event(); + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + /* spawn the thr_wrapper function */ + handle = (HANDLE) _beginthreadex(NULL, stack_size, thr_wrapper, + (LPVOID) &twd, 0, &ID); + if (handle == (HANDLE) 0) { + handle = INVALID_HANDLE_VALUE; + goto error; + } + else { + int spin_count = child_wait_spin_count; + + ETHR_ASSERT(handle != INVALID_HANDLE_VALUE); + + if (!tid->jdata) + CloseHandle(handle); + else + tid->jdata->handle = handle; + + /* Wait for child to initialize... */ + while (1) { + long result; + int err; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + err = (int) result; + goto error; + } + + err = ethr_event_swait(&twd.tse->event, spin_count); + if (err && err != EINTR) + goto error; + spin_count = 0; + } + } + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return 0; + + error: + + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (handle != INVALID_HANDLE_VALUE) { + WaitForSingleObject(handle, INFINITE); + CloseHandle(handle); + } + + if (tid->jdata) { + ethr_mem__.std.free(tid->jdata); + tid->jdata = NULL; + } + + tid->id = ETHR_INVALID_TID_ID; + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return err; +} + +int ethr_thr_join(ethr_tid tid, void **res) +{ + DWORD code; + +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + /* Wait for thread to terminate */ + code = WaitForSingleObject(tid.jdata->handle, INFINITE); + if (code != WAIT_OBJECT_0) + return ethr_win_get_errno__(); + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + if (res) + *res = tid.jdata->res; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +void +ethr_thr_exit(void *res) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ETHR_ASSERT(0); + _endthreadex((unsigned) 0); + } + thr_exit_cleanup(tid, res); + _endthreadex((unsigned) 0); +} + +ethr_tid +ethr_self(void) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + ETHR_ASSERT(0); + return dummy_tid; + } +#endif + /* It is okay for non-ethreads (i.e. native win32 threads) to call + ethr_self(). They will however be returned an invalid tid. */ + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + return dummy_tid; + } + return *tid; +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + /* An invalid tid does not equal any tid, not even an invalid tid */ + return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID; +} + +int +ethr_time_now(ethr_timeval *time) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!time) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + get_curr_time(&time->tv_sec, &time->tv_nsec); + return 0; +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ + DWORD key; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + key = TlsAlloc(); + if (key == TLS_OUT_OF_INDEXES) + return ethr_win_get_errno__(); + *keyp = (ethr_tsd_key) key; + return 0; +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsFree((DWORD) key)) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsSetValue((DWORD) key, (LPVOID) value)) + return ethr_win_get_errno__(); + return 0; +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return (void *) TlsGetValue((DWORD) key); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +ethr_ts_event * +ethr_create_ts_event__(void) +{ + ethr_ts_event *tsep; + ethr_make_ts_event__(&tsep); + return tsep; +} diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index bbc79e9381..0cc315e9be 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -37,21 +37,16 @@ equal_tids/1, mutex/1, try_lock_mutex/1, - recursive_mutex/1, time_now/1, cond_wait/1, - cond_timedwait/1, broadcast/1, detached_thread/1, max_threads/1, - forksafety/1, - vfork/1, tsd/1, spinlock/1, rwspinlock/1, rwmutex/1, - atomic/1, - gate/1]). + atomic/1]). -include_lib("test_server/include/test_server.hrl"). @@ -60,21 +55,16 @@ tests() -> equal_tids, mutex, try_lock_mutex, - recursive_mutex, time_now, cond_wait, - cond_timedwait, broadcast, detached_thread, max_threads, - forksafety, - vfork, tsd, spinlock, rwspinlock, rwmutex, - atomic, - gate]. + atomic]. all(doc) -> []; all(suite) -> tests(). @@ -114,13 +104,6 @@ try_lock_mutex(suite) -> try_lock_mutex(Config) -> run_case(Config, "try_lock_mutex", ""). -recursive_mutex(doc) -> - ["Tests recursive mutexes."]; -recursive_mutex(suite) -> - []; -recursive_mutex(Config) -> - run_case(Config, "recursive_mutex", ""). - time_now(doc) -> ["Tests ethr_time_now by comparing time values with Erlang."]; time_now(suite) -> @@ -169,13 +152,6 @@ cond_wait(suite) -> cond_wait(Config) -> run_case(Config, "cond_wait", ""). -cond_timedwait(doc) -> - ["Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast."]; -cond_timedwait(suite) -> - []; -cond_timedwait(Config) -> - run_case(Config, "cond_timedwait", ""). - broadcast(doc) -> ["Tests that a ethr_cond_broadcast really wakes up all waiting threads"]; broadcast(suite) -> @@ -197,25 +173,6 @@ max_threads(suite) -> max_threads(Config) -> run_case(Config, "max_threads", ""). -forksafety(doc) -> - ["Tests forksafety."]; -forksafety(suite) -> - []; -forksafety(Config) -> - run_case(Config, "forksafety", ""). - -vfork(doc) -> - ["Tests vfork with threads."]; -vfork(suite) -> - case ?t:os_type() of - {unix, osf1} -> - {skip, "vfork() known to hang multi-threaded applications on osf1"}; - _ -> - [] - end; -vfork(Config) -> - run_case(Config, "vfork", ""). - tsd(doc) -> ["Tests thread specific data."]; tsd(suite) -> @@ -251,13 +208,6 @@ atomic(suite) -> atomic(Config) -> run_case(Config, "atomic", ""). -gate(doc) -> - ["Tests gates."]; -gate(suite) -> - []; -gate(Config) -> - run_case(Config, "gate", ""). - %% %% %% Auxiliary functions diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index f779f13c51..7fc71d8047 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -90,10 +90,12 @@ static void print_line(char *frmt,...) print_eol(); } +#if 0 /* Currently not used; silence annoying warning... */ static void print(char *frmt,...) { PRINT_VA_LIST(frmt); } +#endif static void fail(char *frmt,...) { @@ -197,8 +199,8 @@ create_join_thread_test(void) } for (i = 1; i <= CJTT_NO_THREADS; i++) { - int *tres; - res = ethr_thr_join(cjtt_tids[i], (void **) &tres); + void *tres; + res = ethr_thr_join(cjtt_tids[i], &tres); ASSERT(res == 0); ASSERT(tres == &cjtt_res[i]); ASSERT(cjtt_res[i] == i); @@ -216,8 +218,8 @@ create_join_thread_test(void) #define ETT_THREADS 100000 static ethr_tid ett_tids[3]; -static ethr_mutex ett_mutex = ETHR_MUTEX_INITER; -static ethr_cond ett_cond = ETHR_COND_INITER; +static ethr_mutex ett_mutex; +static ethr_cond ett_cond; static int ett_terminate; static void * @@ -234,14 +236,12 @@ static void * ett_thread2(void *unused) { int res; - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); while (!ett_terminate) { res = ethr_cond_wait(&ett_cond, &ett_mutex); ASSERT(res == 0); } - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&ett_mutex); return NULL; } @@ -250,6 +250,10 @@ equal_tids_test(void) { int res, i; + res = ethr_mutex_init(&ett_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&ett_cond); + ASSERT(res == 0); ett_tids[0] = ethr_self(); res = ethr_thr_create(&ett_tids[1], ett_thread, (void *) &ett_tids[1], NULL); @@ -298,13 +302,10 @@ equal_tids_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); ett_terminate = 1; - res = ethr_cond_signal(&ett_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_cond_signal(&ett_cond); + ethr_mutex_unlock(&ett_mutex); res = ethr_thr_join(ett_tids[1], NULL); ASSERT(res == 0); @@ -321,17 +322,14 @@ equal_tids_test(void) * Tests mutexes. */ -static ethr_mutex mt_mutex = ETHR_MUTEX_INITER; +static ethr_mutex mt_mutex; static int mt_data; void * mt_thread(void *unused) { - int res; - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Aux thread locked mutex"); ASSERT(mt_data == 0); @@ -345,8 +343,7 @@ mt_thread(void *unused) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Aux thread unlocked mutex"); return NULL; @@ -356,18 +353,18 @@ mt_thread(void *unused) static void mutex_test(void) { - int do_restart = 1; int res; ethr_tid tid; - print_line("Running test with statically initialized mutex"); + print_line("Trying to initialize mutex"); + res = ethr_mutex_init(&mt_mutex); + ASSERT(res == 0); + print_line("Initialized mutex"); - restart: mt_data = 0; print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 0); @@ -383,8 +380,7 @@ mutex_test(void) ASSERT(mt_data == 0); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); print_line("Main thread goes to sleep for 1 second"); @@ -392,8 +388,7 @@ mutex_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 1); @@ -404,8 +399,7 @@ mutex_test(void) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); res = ethr_thr_join(tid, NULL); @@ -416,20 +410,6 @@ mutex_test(void) ASSERT(res == 0); print_line("Main thread destroyed mutex"); - if (do_restart) { - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_mutex_init(&mt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - - } - } /* @@ -438,9 +418,9 @@ mutex_test(void) * Tests try lock mutex operation. */ -static ethr_mutex tlmt_mtx1 = ETHR_MUTEX_INITER; -static ethr_mutex tlmt_mtx2 = ETHR_MUTEX_INITER; -static ethr_cond tlmt_cnd2 = ETHR_COND_INITER; +static ethr_mutex tlmt_mtx1; +static ethr_mutex tlmt_mtx2; +static ethr_cond tlmt_cnd2; static int tlmt_mtx1_locked; static int tlmt_mtx1_do_unlock; @@ -450,32 +430,24 @@ tlmt_thread(void *unused) { int res; - res = ethr_mutex_lock(&tlmt_mtx1); - ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx1); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (!tlmt_mtx1_do_unlock) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); + ethr_mutex_unlock(&tlmt_mtx1); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 0; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); + ethr_mutex_unlock(&tlmt_mtx2); return NULL; } @@ -486,48 +458,49 @@ try_lock_mutex_test(void) int i, res; ethr_tid tid; + res = ethr_mutex_init(&tlmt_mtx1); + ASSERT(res == 0); + res = ethr_mutex_init(&tlmt_mtx2); + ASSERT(res == 0); + res = ethr_cond_init(&tlmt_cnd2); + ASSERT(res == 0); + tlmt_mtx1_locked = 0; tlmt_mtx1_do_unlock = 0; res = ethr_thr_create(&tid, tlmt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); while (!tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); for (i = 0; i < 10; i++) { res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == EBUSY); } - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_do_unlock = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx1); res = ethr_thr_join(tid, NULL); ASSERT(res == 0); @@ -541,159 +514,6 @@ try_lock_mutex_test(void) } /* - * The recursive mutex test case. - * - * Tests recursive mutexes. - */ - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static ethr_mutex rmt_mutex -#ifdef ETHR_REC_MUTEX_INITER - = ETHR_REC_MUTEX_INITER -#endif - ; -static int rmt_data; - -void * -rmt_thread(void *unused) -{ - int res; - - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread locked mutex"); - - ASSERT(rmt_data == 0); - - rmt_data = 1; - print_line("Aux thread wrote"); - - print_line("Aux thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Aux thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread unlocked mutex"); - - return NULL; -} - -#endif - -static void -recursive_mutex_test(void) -{ -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - int do_restart = 1; - int res; - ethr_tid tid; - -#ifdef ETHR_REC_MUTEX_INITER - print_line("Running test with statically initialized mutex"); -#else - goto dynamic_init; -#endif - - restart: - rmt_data = 0; - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - print_line("Main thread tries to lock mutex again"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex again"); - - ASSERT(rmt_data == 0); - - print_line("Main thread about to create aux thread"); - res = ethr_thr_create(&tid, rmt_thread, NULL, NULL); - ASSERT(res == 0); - print_line("Main thread created aux thread"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex again"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - ASSERT(rmt_data == 1); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - print_line("Main thread joined aux thread"); - - res = ethr_mutex_destroy(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread destroyed mutex"); - - if (do_restart) { -#ifndef ETHR_REC_MUTEX_INITER - dynamic_init: -#endif - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_rec_mutex_init(&rmt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - } - -#ifndef ETHR_REC_MUTEX_INITER - succeed("Static initializer for recursive mutexes not supported"); -#endif - -#else /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - skip("Recursive mutexes not supported"); -#endif /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ -} - -/* * The time now test. * * Tests ethr_time_now by comparing time values with Erlang. @@ -763,106 +583,82 @@ time_now_test(void) */ -static ethr_mutex cwt_mutex = ETHR_MUTEX_INITER; -static ethr_cond cwt_cond = ETHR_COND_INITER; +static ethr_mutex cwt_mutex; +static ethr_cond cwt_cond; static int cwt_counter; void * -cwt_thread(void *is_timedwait_test_ptr) +cwt_thread(void *unused) { - int use_timedwait = *((int *) is_timedwait_test_ptr); int res; - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); - - if (use_timedwait) { - ethr_timeval tv; - res = ethr_time_now(&tv); - ASSERT(res == 0); - tv.tv_sec += 3600; /* Make sure we won't time out */ + ethr_mutex_lock(&cwt_mutex); - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tv); - } while (res == EINTR); - ASSERT(res == 0); - } - else { - do { - res = ethr_cond_wait(&cwt_cond, &cwt_mutex); - } while (res == EINTR); - ASSERT(res == 0); - } + do { + res = ethr_cond_wait(&cwt_cond, &cwt_mutex); + } while (res == EINTR); + ASSERT(res == 0); cwt_counter++; - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); return NULL; } static void -cond_wait_test(int is_timedwait_test) +cond_wait_test(void) { - int do_restart = !is_timedwait_test; ethr_tid tid1, tid2; int res; - if (!is_timedwait_test) - print_line("Running test with statically initialized mutex and cond"); + res = ethr_mutex_init(&cwt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&cwt_cond); + ASSERT(res == 0); - restart: /* Wake with signal */ cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_signal(&cwt_cond); /* Wake one thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake one thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 1); - res = ethr_cond_signal(&cwt_cond); /* Wake the other thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake the other thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 1); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -875,35 +671,30 @@ cond_wait_test(int is_timedwait_test) cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ - ASSERT(res == 0); + ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ do_sleep(1); /* Make sure awakened threads wait on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened threads proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -916,113 +707,6 @@ cond_wait_test(int is_timedwait_test) res = ethr_cond_destroy(&cwt_cond); ASSERT(res == 0); - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } -} - -/* - * The cond timedwait test case. - * - * Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast. - */ - -#define CTWT_MAX_TIME_DIFF 100000 - -static long -ctwt_check_timeout(long to) -{ - int res; - ethr_timeval tva, tvb; - long diff, abs_diff; - - res = ethr_time_now(&tva); - ASSERT(res == 0); - - tva.tv_sec += to / 1000; - tva.tv_nsec += (to % 1000) * 1000000; - if (tva.tv_nsec >= 1000000000) { - tva.tv_sec++; - tva.tv_nsec -= 1000000000; - ASSERT(tva.tv_nsec < 1000000000); - } - - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tva); - } while (res == EINTR); - ASSERT(res == ETIMEDOUT); - - res = ethr_time_now(&tvb); - ASSERT(res == 0); - - diff = (tvb.tv_sec - tva.tv_sec) * 1000000; - diff += (tvb.tv_nsec - tva.tv_nsec)/1000; - - print("Timeout=%ld; ", to); - print("tva.tv_sec=%ld tva.tv_nsec=%ld; ", tva.tv_sec, tva.tv_nsec); - print("tvb.tv_sec=%ld tvb.tv_nsec=%ld; ", tvb.tv_sec, tvb.tv_nsec); - print_line("diff (tvb - tva) = %ld us", diff); - - abs_diff = (long) abs((int) diff); - - ASSERT(CTWT_MAX_TIME_DIFF >= abs_diff); - return abs_diff; -} - -static void -cond_timedwait_test(void) -{ - int do_restart = 1; - long abs_diff, max_abs_diff = 0; - int res; - -#define CTWT_UPD_MAX_DIFF if (abs_diff > max_abs_diff) max_abs_diff = abs_diff; - - print_line("Running test with statically initialized mutex and cond"); - - print_line("CTWT_MAX_TIME_DIFF=%d", CTWT_MAX_TIME_DIFF); - - restart: - - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); - - abs_diff = ctwt_check_timeout(300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(700); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(1000); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(2300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(5100); - CTWT_UPD_MAX_DIFF; - - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); - - cond_wait_test(1); - - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } - - print_line("Max absolute diff = %d us", max_abs_diff); - succeed("Max absolute diff = %d us", max_abs_diff); - -#undef CTWT_UPD_MAX_DIFF } /* @@ -1037,9 +721,9 @@ cond_timedwait_test(void) static int bct_woken = 0; static int bct_waiting = 0; static int bct_done = 0; -static ethr_mutex bct_mutex = ETHR_MUTEX_INITER; -static ethr_cond bct_cond = ETHR_COND_INITER; -static ethr_cond bct_cntrl_cond = ETHR_COND_INITER; +static ethr_mutex bct_mutex; +static ethr_cond bct_cond; +static ethr_cond bct_cntrl_cond; static void * @@ -1047,30 +731,24 @@ bct_thread(void *unused) { int res; - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); while (!bct_done) { bct_waiting++; - if (bct_waiting == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_waiting == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); do { res = ethr_cond_wait(&bct_cond, &bct_mutex); } while (res == EINTR); ASSERT(res == 0); bct_woken++; - if (bct_woken == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_woken == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); } - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); return NULL; } @@ -1081,14 +759,20 @@ broadcast_test(void) int res, i; ethr_tid tid[BCT_THREADS]; + res = ethr_mutex_init(&bct_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cntrl_cond); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cond); + ASSERT(res == 0); + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_create(&tid[i], bct_thread, NULL, NULL); ASSERT(res == 0); } - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); for (i = 0; i < BCT_NO_OF_WAITS; i++) { @@ -1101,8 +785,7 @@ broadcast_test(void) bct_woken = 0; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); while (bct_woken != BCT_THREADS) { res = ethr_cond_wait(&bct_cntrl_cond, &bct_mutex); @@ -1114,13 +797,11 @@ broadcast_test(void) bct_done = 1; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); - for (i = 0; i < BCT_THREADS - 1; i++) { + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_join(tid[i], NULL); ASSERT(res == 0); } @@ -1143,26 +824,22 @@ broadcast_test(void) #define DT_THREADS (50*1024) #define DT_BATCH_SIZE 64 -static ethr_mutex dt_mutex = ETHR_MUTEX_INITER; -static ethr_cond dt_cond = ETHR_COND_INITER; +static ethr_mutex dt_mutex; +static ethr_cond dt_cond; static int dt_count; static int dt_limit; static void * dt_thread(void *unused) { - int res; - - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); dt_count++; if (dt_count >= dt_limit) ethr_cond_signal(&dt_cond); - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); return NULL; } @@ -1174,6 +851,11 @@ detached_thread_test(void) ethr_tid tid[DT_BATCH_SIZE]; int i, j, res; + res = ethr_mutex_init(&dt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&dt_cond); + ASSERT(res == 0); + thr_opts.detached = 1; dt_count = 0; dt_limit = 0; @@ -1187,14 +869,12 @@ detached_thread_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); while (dt_count < dt_limit) { res = ethr_cond_wait(&dt_cond, &dt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); print_line("dt_count = %d", dt_count); } @@ -1211,8 +891,8 @@ detached_thread_test(void) #define MTT_TIMES 10 static int mtt_terminate; -static ethr_mutex mtt_mutex = ETHR_MUTEX_INITER; -static ethr_cond mtt_cond = ETHR_COND_INITER; +static ethr_mutex mtt_mutex; +static ethr_cond mtt_cond; static char mtt_string[22*MTT_TIMES]; /* 22 is enough for ", %d" */ @@ -1220,16 +900,14 @@ void *mtt_thread(void *unused) { int res; - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); while (!mtt_terminate) { res = ethr_cond_wait(&mtt_cond, &mtt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); return NULL; } @@ -1265,16 +943,13 @@ mtt_create_join_threads(void) print_line("%d = ethr_thr_create()", res); print_line("Number of created threads: %d", no_threads); - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); mtt_terminate = 1; - res = ethr_cond_broadcast(&mtt_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&mtt_cond); - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); while (ix) { res = ethr_thr_join(tids[--ix], NULL); @@ -1292,9 +967,14 @@ mtt_create_join_threads(void) static void max_threads_test(void) { - int no_threads[MTT_TIMES], i, up, down, eq; + int no_threads[MTT_TIMES], i, up, down, eq, res; char *str; + res = ethr_mutex_init(&mtt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&mtt_cond); + ASSERT(res == 0); + for (i = 0; i < MTT_TIMES; i++) { no_threads[i] = mtt_create_join_threads(); } @@ -1326,272 +1006,6 @@ max_threads_test(void) } - -/* - * The forksafety test case. - * - * Tests forksafety. - */ -#ifdef __WIN32__ -#define NO_FORK_PRESENT -#endif - -#ifndef NO_FORK_PRESENT - -static ethr_mutex ft_test_inner_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_test_outer_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_go_mutex = ETHR_MUTEX_INITER; -static ethr_cond ft_go_cond = ETHR_COND_INITER; -static int ft_go; -static int ft_have_forked; - -static void * -ft_thread(void *unused) -{ - int res; - - res = ethr_mutex_lock(&ft_test_outer_mutex); - ASSERT(res == 0); - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - ft_go = 1; - res = ethr_cond_signal(&ft_go_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_lock(&ft_test_inner_mutex); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_test_inner_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_unlock(&ft_test_outer_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(ft_have_forked); - - - return NULL; -} - -#endif /* #ifndef NO_FORK_PRESENT */ - -static void -forksafety_test(void) -{ -#ifdef NO_FORK_PRESENT - skip("No fork() present; nothing to test"); -#elif defined(DEBUG) - skip("Doesn't work in debug build"); -#else - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - ethr_tid tid; - int res; - int fds[2]; - - - res = ethr_mutex_set_forksafe(&ft_test_inner_mutex); - if (res == ENOTSUP) { - skip("Forksafety not supported on this platform!"); - } - ASSERT(res == 0); - res = ethr_mutex_set_forksafe(&ft_test_outer_mutex); - ASSERT(res == 0); - - - res = pipe(fds); - ASSERT(res == 0); - - ft_go = 0; - ft_have_forked = 0; - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, ft_thread, NULL, NULL); - ASSERT(res == 0); - - do { - res = ethr_cond_wait(&ft_go_cond, &ft_go_mutex); - } while (res == EINTR || !ft_go); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - res = fork(); - ft_have_forked = 1; - if (res == 0) { - close(fds[0]); - res = ethr_mutex_lock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_lock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = ethr_mutex_destroy(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_destroy(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = (int) write(fds[1], (void *) snd_msg, sizeof(snd_msg)); - if (res != sizeof(snd_msg)) - _exit(1); - close(fds[1]); - _exit(0); - } - ASSERT(res > 0); - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); -#endif -} - - -/* - * The vfork test case. - * - * Tests vfork with threads. - */ - -#ifdef __WIN32__ -#define NO_VFORK_PRESENT -#endif - -#ifndef NO_VFORK_PRESENT - -#undef vfork - -static ethr_mutex vt_mutex = ETHR_MUTEX_INITER; - -static void * -vt_thread(void *vprog) -{ - char *prog = (char *) vprog; - int res; - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - int fds[2]; - char closefd[20]; - char writefd[20]; - - res = pipe(fds); - ASSERT(res == 0); - - res = sprintf(closefd, "%d", fds[0]); - ASSERT(res <= 20); - res = sprintf(writefd, "%d", fds[1]); - ASSERT(res <= 20); - - print("parent: About to vfork and execute "); - print("execlp(\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", NULL)", - prog, prog, "vfork", "exec", snd_msg, closefd, writefd); - print_line(" in child"); - res = vfork(); - if (res == 0) { - execlp(prog, prog, "vfork", "exec", snd_msg, closefd, writefd, NULL); - _exit(1); - } - ASSERT(res > 0); - - print_line("parent: I'm back"); - - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - print_line("parent: %d = read()", res); - print_line("parent: rec_msg=\"%s\"", rec_msg); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); - - return NULL; -} - -#endif /* #ifndef NO_VFORK_PRESENT */ - -static void -vfork_test(int argc, char *argv[]) -{ -#ifdef NO_VFORK_PRESENT - skip("No vfork() present; nothing to test"); -#else - int res; - ethr_tid tid; - - if (argc == 6 && strcmp("exec", argv[2]) == 0) { - /* We are child after vfork() and execlp() ... */ - - char *snd_msg; - int closefd; - int writefd; - - snd_msg = argv[3]; - closefd = atoi(argv[4]); - writefd = atoi(argv[5]); - - print_line("child: snd_msg=\"%s\"; closefd=%d writefd=%d", - snd_msg, closefd, writefd); - - close(closefd); - - res = (int) write(writefd, (void *) snd_msg, strlen(snd_msg)+1); - print_line("child: %d = write()", res); - if (res != strlen(snd_msg)+1) - exit(1); - close(writefd); - print_line("child: bye"); - exit(0); - } - ASSERT(argc == 2); - - res = ethr_mutex_set_forksafe(&vt_mutex); - ASSERT(res == 0 || res == ENOTSUP); - res = ethr_mutex_lock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, vt_thread, (void *) argv[0], NULL); - ASSERT(res == 0); - - do_sleep(1); - - res = ethr_mutex_unlock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - - res = ethr_mutex_destroy(&vt_mutex); - ASSERT(res == 0); -#endif -} - - /* * The tsd test case. * @@ -1651,11 +1065,8 @@ static int st_data; void * st_thread(void *unused) { - int res; - print_line("Aux thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Aux thread locked spinlock"); ASSERT(st_data == 0); @@ -1669,8 +1080,7 @@ st_thread(void *unused) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Aux thread unlocked spinlock"); return NULL; @@ -1691,8 +1101,7 @@ spinlock_test(void) st_data = 0; print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 0); @@ -1708,8 +1117,7 @@ spinlock_test(void) ASSERT(st_data == 0); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); print_line("Main thread goes to sleep for 1 second"); @@ -1717,8 +1125,7 @@ spinlock_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 1); @@ -1729,8 +1136,7 @@ spinlock_test(void) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); res = ethr_thr_join(tid, NULL); @@ -1757,23 +1163,19 @@ void * rwst_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Aux thread read locked rwspinlock"); ASSERT(rwst_data == 4711); print_line("Aux thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Aux thread read unlocked rwspinlock"); print_line("Aux thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Aux thread write locked rwspinlock"); data = ++rwst_data; @@ -1787,8 +1189,7 @@ rwst_thread(void *unused) ++rwst_data; print_line("Aux thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Aux thread write unlocked rwspinlock"); return NULL; @@ -1810,8 +1211,7 @@ rwspinlock_test(void) rwst_data = 4711; print_line("Main thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Main thread read locked rwspinlock"); ASSERT(rwst_data == 4711); @@ -1828,13 +1228,11 @@ rwspinlock_test(void) ASSERT(rwst_data == 4711); print_line("Main thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Main thread read unlocked rwspinlock"); print_line("Main thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Main thread write locked rwspinlock"); data = ++rwst_data; @@ -1847,8 +1245,7 @@ rwspinlock_test(void) ++rwst_data; print_line("Main thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Main thread write unlocked rwspinlock"); res = ethr_thr_join(tid, NULL); @@ -1875,23 +1272,19 @@ void * rwmt_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Aux thread read locked rwmutex"); ASSERT(rwmt_data == 4711); print_line("Aux thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Aux thread read unlocked rwmutex"); print_line("Aux thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Aux thread write locked rwmutex"); data = ++rwmt_data; @@ -1905,8 +1298,7 @@ rwmt_thread(void *unused) ++rwmt_data; print_line("Aux thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Aux thread write unlocked rwmutex"); return NULL; @@ -1928,8 +1320,7 @@ rwmutex_test(void) rwmt_data = 4711; print_line("Main thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Main thread read locked rwmutex"); ASSERT(rwmt_data == 4711); @@ -1946,13 +1337,11 @@ rwmutex_test(void) ASSERT(rwmt_data == 4711); print_line("Main thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Main thread read unlocked rwmutex"); print_line("Main thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Main thread write locked rwmutex"); data = ++rwmt_data; @@ -1965,8 +1354,7 @@ rwmutex_test(void) ++rwmt_data; print_line("Main thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Main thread write unlocked rwmutex"); res = ethr_thr_join(tid, NULL); @@ -1998,65 +1386,53 @@ static ethr_atomic_t at_data; void * at_thread(void *unused) { - int res, i; + int i; long val, go; - res = ethr_atomic_inctest(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_ready); ASSERT(val > 0); ASSERT(val <= AT_THREADS); do { - res = ethr_atomic_read(&at_go, &go); - ASSERT(res == 0); + go = ethr_atomic_read(&at_go); } while (!go); for (i = 0; i < AT_ITER; i++) { - res = ethr_atomic_or_old(&at_data, at_set_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_bor(&at_data, at_set_val); ASSERT(val >= (i == 0 ? 0 : at_set_val) + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_and_old(&at_data, ~at_rm_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_band(&at_data, ~at_rm_val); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inctest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_data); ASSERT(val > at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_dectest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_dec_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inc(&at_data); - ASSERT(res == 0); + ethr_atomic_inc(&at_data); - res = ethr_atomic_dec(&at_data); - ASSERT(res == 0); + ethr_atomic_dec(&at_data); - res = ethr_atomic_addtest(&at_data, (long) 4711, &val); - ASSERT(res == 0); + val = ethr_atomic_add_read(&at_data, (long) 4711); ASSERT(val >= at_set_val + (long) 2*4711); ASSERT(val <= at_max_val); - res = ethr_atomic_add(&at_data, (long) -4711); - ASSERT(res == 0); + ethr_atomic_add(&at_data, (long) -4711); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); } - res = ethr_atomic_inc(&at_done); - ASSERT(res == 0); + ethr_atomic_inc(&at_done); return NULL; } @@ -2069,14 +1445,13 @@ atomic_test(void) ethr_tid tid[AT_THREADS]; ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; - if (sizeof(long) > 4) { +#if ETHR_SIZEOF_PTR > 4 at_rm_val = ((long) 1) << 57; at_set_val = ((long) 1) << 60; - } - else { +#else at_rm_val = ((long) 1) << 27; at_set_val = ((long) 1) << 30; - } +#endif at_max_val = at_set_val + at_rm_val + ((long) AT_THREADS + 1) * 4711; data_init = at_rm_val + (long) 4711; @@ -2085,22 +1460,15 @@ atomic_test(void) thr_opts.detached = 1; print_line("Initializing"); - res = ethr_atomic_init(&at_ready, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_go, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_done, data_init); - ASSERT(res == 0); - res = ethr_atomic_init(&at_data, data_init); - ASSERT(res == 0); + ethr_atomic_init(&at_ready, 0); + ethr_atomic_init(&at_go, 0); + ethr_atomic_init(&at_done, data_init); + ethr_atomic_init(&at_data, data_init); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val == data_init); - res = ethr_atomic_set(&at_done, 0); - ASSERT(res == 0); - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + ethr_atomic_set(&at_done, 0); + val = ethr_atomic_read(&at_done); ASSERT(val == 0); print_line("Creating threads"); @@ -2111,31 +1479,27 @@ atomic_test(void) print_line("Waiting for threads to ready up"); do { - res = ethr_atomic_read(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_ready); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Letting threads loose"); - res = ethr_atomic_xchg(&at_go, 17, &val); - ASSERT(res == 0); + val = ethr_atomic_xchg(&at_go, 17); ASSERT(val == 0); - res = ethr_atomic_read(&at_go, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_go); ASSERT(val == 17); print_line("Waiting for threads to finish"); do { - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_done); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Checking result"); - res = ethr_atomic_read(&at_data, &val); + val = ethr_atomic_read(&at_data); ASSERT(res == 0); ASSERT(val == data_final); print_line("Result ok"); @@ -2143,190 +1507,6 @@ atomic_test(void) } -/* - * The gate test case. - * - * Tests gates. - */ - -#define GT_THREADS 10 - -static ethr_atomic_t gt_wait1; -static ethr_atomic_t gt_wait2; -static ethr_atomic_t gt_done; - -static ethr_gate gt_gate1; -static ethr_gate gt_gate2; - -void * -gt_thread(void *thr_no) -{ - int no = (int)(long) thr_no; - int swait = no % 2 == 0; - int res; - long done; - - - do { - - res = ethr_atomic_inc(>_wait1); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate1, INT_MAX); - else - res = ethr_gate_wait(>_gate1); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait1); - ASSERT(res == 0); - - res = ethr_atomic_inc(>_wait2); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate2, INT_MAX); - else - res = ethr_gate_wait(>_gate2); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait2); - ASSERT(res == 0); - - res = ethr_atomic_read(>_done, &done); - ASSERT(res == 0); - } while (!done); - return NULL; -} - - -static void -gate_test(void) -{ - long val; - int res, i; - ethr_tid tid[GT_THREADS]; - - print_line("Initializing"); - res = ethr_atomic_init(>_wait1, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_wait2, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_done, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate2); - ASSERT_EQ(res, 0, "%d"); - - print_line("Creating threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_create(&tid[i], gt_thread, (void *) i, NULL); - ASSERT_EQ(res, 0, "%d"); - } - - print_line("Waiting for threads to ready up"); - do { - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT(0 <= val && val <= GT_THREADS); - } while (val != GT_THREADS); - - print_line("Testing"); - - res = ethr_gate_let_through(>_gate1, 8); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 8)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 8, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 8, "%ld"); - - res = ethr_gate_let_through(>_gate2, 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_gate_let_through(>_gate1, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == GT_THREADS)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS, "%ld"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_atomic_set(>_done, 1); - ASSERT_EQ(res, 0, "%d"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_let_through(>_gate1, GT_THREADS - 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM(((res = ethr_atomic_read(>_wait1, &val)) != 0 - || (val == 0 - && ((res = ethr_atomic_read(>_wait2, &val)) != 0 - || val == 0))), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - print_line("Joining threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_join(tid[i], NULL); - ASSERT_EQ(res, 0, "%d"); - } - - res = ethr_gate_destroy(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_destroy(>_gate2); - ASSERT_EQ(res, 0, "%d"); - -} - #endif /* #ifndef ETHR_NO_THREAD_LIB */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -2342,14 +1522,12 @@ main(int argc, char *argv[]) #ifndef ETHR_NO_THREAD_LIB { char *testcase; - int res; send_my_pid(); testcase = argv[1]; - res = ethr_init(NULL); - if (res != 0) + if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0) fail("Failed to initialize the ethread library"); if (strcmp(testcase, "create_join_thread") == 0) @@ -2360,24 +1538,16 @@ main(int argc, char *argv[]) mutex_test(); else if (strcmp(testcase, "try_lock_mutex") == 0) try_lock_mutex_test(); - else if (strcmp(testcase, "recursive_mutex") == 0) - recursive_mutex_test(); else if (strcmp(testcase, "time_now") == 0) time_now_test(); else if (strcmp(testcase, "cond_wait") == 0) - cond_wait_test(0); - else if (strcmp(testcase, "cond_timedwait") == 0) - cond_timedwait_test(); + cond_wait_test(); else if (strcmp(testcase, "broadcast") == 0) broadcast_test(); else if (strcmp(testcase, "detached_thread") == 0) detached_thread_test(); else if (strcmp(testcase, "max_threads") == 0) max_threads_test(); - else if (strcmp(testcase, "forksafety") == 0) - forksafety_test(); - else if (strcmp(testcase, "vfork") == 0) - vfork_test(argc, argv); else if (strcmp(testcase, "tsd") == 0) tsd_test(); else if (strcmp(testcase, "spinlock") == 0) @@ -2388,8 +1558,6 @@ main(int argc, char *argv[]) rwmutex_test(); else if (strcmp(testcase, "atomic") == 0) atomic_test(); - else if (strcmp(testcase, "gate") == 0) - gate_test(); else skip("Test case \"%s\" not implemented yet", testcase); diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index ee1befc882..5df60a92e5 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -1039,15 +1039,22 @@ ets:select(Table,MatchSpec),</code> the owner terminates.</p> </item> <item> + <marker id="new_2_write_concurrency"></marker> <p><c>{write_concurrency,bool()}</c> - Performance tuning. Default is <c>false</c>, which means that the table - is optimized towards concurrent read access. An operation that + Performance tuning. Default is <c>false</c>. An operation that mutates (writes to) the table will obtain exclusive access, blocking any concurrent access of the same table until finished. If set to <c>true</c>, the table is optimized towards concurrent write access. Different objects of the same table can be mutated (and read) by concurrent processes. This is achieved to some degree at the expense of single access and concurrent reader performance. + The <c>write_concurrency</c> option can be combined with the + <seealso marker="#new_2_read_concurrency">read_concurrency</seealso> + option. You typically want to combine these when large concurrent + read bursts and large concurrent write bursts are common (see the + documentation of the + <seealso marker="#new_2_read_concurrency">read_concurrency</seealso> + option for more information). Note that this option does not change any guarantees about <seealso marker="#concurrency">atomicy and isolation</seealso>. Functions that makes such promises over several objects (like @@ -1055,6 +1062,29 @@ ets:select(Table,MatchSpec),</code> <p>Table type <c>ordered_set</c> is not affected by this option in current implementation.</p> </item> + <item> + <marker id="new_2_read_concurrency"></marker> + <p><c>{read_concurrency,bool()}</c> + Performance tuning. Default is <c>false</c>. When set to + <c>true</c>, the table is optimized for concurrent read + operations. When this option is enabled on a runtime system with + SMP support, read operations become much cheaper; especially on + systems with multiple physical processors. However, switching + between read and write operations becomes more expensive. You + typically want to enable this option when concurrent read + operations are much more frequent than write operations, or when + concurrent reads and writes comes in large read and write + bursts (i.e., lots of reads not interrupted by writes, and lots + of writes not interrupted by reads). You typically do + <em>not</em> want to enable this option when the common access + pattern is a few read operations interleaved with a few write + operations repeatedly. In this case you will get a performance + degradation by enabling this option. The <c>read_concurrency</c> + option can be combined with the + <seealso marker="#new_2_write_concurrency">write_concurrency</seealso> + option. You typically want to combine these when large concurrent + read bursts and large concurrent write bursts are common.</p> + </item> </list> </desc> </func> diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 13c87ca005..1f68c6b8c4 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -33,7 +33,7 @@ -export([misc/1, dups/1, misc1/1, safe_fixtable/1, info/1, tab2list/1]). -export([files/1, tab2file/1, tab2file2/1, tab2file3/1, tabfile_ext1/1, tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1]). --export([heavy/1, heavy_lookup/1, heavy_lookup_element/1]). +-export([heavy/1, heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). -export([lookup_element/1, lookup_element_mult/1]). -export([fold/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). @@ -89,7 +89,8 @@ match_delete_do/1, match_delete3_do/1, firstnext_do/1, slot_do/1, match1_do/1, match2_do/1, match_object_do/1, match_object2_do/1, misc1_do/1, safe_fixtable_do/1, info_do/1, dups_do/1, heavy_lookup_do/1, - heavy_lookup_element_do/1, member_do/1, otp_5340_do/1, otp_7665_do/1, meta_wb_do/1 + heavy_lookup_element_do/1, member_do/1, otp_5340_do/1, otp_7665_do/1, meta_wb_do/1, + do_heavy_concurrent/1 ]). -include("test_server.hrl"). @@ -3877,7 +3878,7 @@ make_sub_binary(List, Num) when is_list(List) -> {_,B} = split_binary(Bin, N+1), B. -heavy(suite) -> [heavy_lookup, heavy_lookup_element]. +heavy(suite) -> [heavy_lookup, heavy_lookup_element, heavy_concurrent]. %% Lookup stuff like crazy... heavy_lookup(doc) -> ["Performs multiple lookups for every key ", @@ -3940,6 +3941,44 @@ do_lookup_element(Tab, N, M) -> end. +heavy_concurrent(Config) -> + repeat_for_opts(do_heavy_concurrent). + +do_heavy_concurrent(Opts) -> + ?line Size = 20000, + ?line EtsMem = etsmem(), + ?line Tab = ets:new(blupp, [set, public, {keypos, 2} | Opts]), + ?line ok = fill_tab2(Tab, 0, Size), + ?line Procs = lists:map( + fun (N) -> + spawn_link( + fun () -> + do_heavy_concurrent_proc(Tab, Size, N) + end) + end, + lists:seq(1, 500)), + ?line lists:foreach(fun (P) -> + M = erlang:monitor(process, P), + receive + {'DOWN', Mon, process, P, _} -> + ok + end + end, + Procs), + ?line true = ets:delete(Tab), + ?line verify_etsmem(EtsMem). + +do_heavy_concurrent_proc(_Tab, 0, _Offs) -> + done; +do_heavy_concurrent_proc(Tab, N, Offs) when (N+Offs) rem 100 == 0 -> + Data = {"here", are, "S O M E ", data, "toooooooooooooooooo", insert, + make_ref(), make_ref(), make_ref()}, + true=ets:insert(Tab, {{self(),Data}, N}), + do_heavy_concurrent_proc(Tab, N-1, Offs); +do_heavy_concurrent_proc(Tab, N, Offs) -> + _ = ets:lookup(Tab, N), + do_heavy_concurrent_proc(Tab, N-1, Offs). + fold(suite) -> [foldl_ordered, foldr_ordered, foldl, foldr, fold_empty]. @@ -5336,7 +5375,7 @@ only_if_smp(Schedulers, Func) -> %% Repeat test function with different combination of table options %% repeat_for_opts(F) -> - repeat_for_opts(F, [write_concurrency]). + repeat_for_opts(F, [write_concurrency, read_concurrency]). repeat_for_opts(F, OptGenList) when is_atom(F) -> repeat_for_opts(fun(Opts) -> ?MODULE:F(Opts) end, OptGenList); @@ -5356,6 +5395,7 @@ repeat_for_opts(F, [Atom | Tail], AccList) when is_atom(Atom) -> repeat_for_opts(F, [repeat_for_opts_atom2list(Atom) | Tail ], AccList). repeat_for_opts_atom2list(all_types) -> [set,ordered_set,bag,duplicate_bag]; -repeat_for_opts_atom2list(write_concurrency) -> [{write_concurrency,false},{write_concurrency,true}]. +repeat_for_opts_atom2list(write_concurrency) -> [{write_concurrency,false},{write_concurrency,true}]; +repeat_for_opts_atom2list(read_concurrency) -> [{read_concurrency,false},{read_concurrency,true}]. diff --git a/lib/tools/c_src/Makefile.in b/lib/tools/c_src/Makefile.in index e6b76e2238..65a7f5f424 100644 --- a/lib/tools/c_src/Makefile.in +++ b/lib/tools/c_src/Makefile.in @@ -128,12 +128,13 @@ EMEM_ERTS_LIB=erts_r$(TYPEMARKER) endif +EMEM_ETHR_LIBS=$(subst -l$(ETHR_LIB_NAME),-l$(ETHR_LIB_NAME)$(TYPEMARKER),$(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ETHR_LIBS))) + EMEM_LIBS = $(LIBS) \ -L$(ERL_TOP)/erts/lib/$(TARGET) \ -L$(ERL_TOP)/erts/lib/internal/$(TARGET) \ -l$(EMEM_ERTS_LIB) \ - -l$(ETHR_LIB_NAME)$(TYPEMARKER) \ - $(ETHR_X_LIBS) + $(EMEM_ETHR_LIBS) EMEM_OBJS = $(addprefix $(EMEM_OBJ_DIR)/,$(notdir $(EMEM_SRCS:.c=.o))) diff --git a/lib/tools/c_src/erl_memory.c b/lib/tools/c_src/erl_memory.c index a0e139f059..872d55e789 100644 --- a/lib/tools/c_src/erl_memory.c +++ b/lib/tools/c_src/erl_memory.c @@ -1,22 +1,22 @@ -/* ``The contents of this file are subject to the Erlang Public License, +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * + * retrieved online at http://www.erlang.org/. + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ + * + * %CopyrightEnd% */ - /* * Description: * @@ -281,17 +281,13 @@ mutex_destroy(ethr_mutex *mtx) static INLINE void mutex_lock(ethr_mutex *mtx) { - int res = ethr_mutex_lock(mtx); - if (res) - error_msg(res, "Mutex lock"); + ethr_mutex_lock(mtx); } static INLINE void mutex_unlock(ethr_mutex *mtx) { - int res = ethr_mutex_unlock(mtx); - if (res) - error_msg(res, "Mutex unlock"); + ethr_mutex_unlock(mtx); } static INLINE void @@ -314,16 +310,14 @@ static INLINE void cond_wait(ethr_cond *cnd, ethr_mutex *mtx) { int res = ethr_cond_wait(cnd, mtx); - if (res) + if (res != 0 && res != EINTR) error_msg(res, "Cond wait"); } static INLINE void cond_signal(ethr_cond *cnd) { - int res = ethr_cond_signal(cnd); - if (res) - error_msg(res, "Cond signal"); + ethr_cond_signal(cnd); } @@ -2774,7 +2768,7 @@ main(int argc, char *argv[]) exit(1); } - if (ethr_init(NULL) != 0) { + if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0) { fprintf(stderr, "emem: failed to initialize thread package\n"); exit(1); } |