diff options
Diffstat (limited to 'erts')
281 files changed, 26324 insertions, 26852 deletions
diff --git a/erts/Makefile b/erts/Makefile index 12d2ec57a8..0393ccc759 100644 --- a/erts/Makefile +++ b/erts/Makefile @@ -34,7 +34,7 @@ ERTSDIRS += start_scripts endif .PHONY: all -all: $(FLAVORS) +all: smp .PHONY: docs docs: @@ -44,20 +44,15 @@ docs: debug opt lcnt clean: $(V_at)for d in emulator $(ERTSDIRS); do \ if test -d $$d; then \ - ( cd $$d && $(MAKE) $@ FLAVOR=$(FLAVOR) ) || exit $$? ; \ + ( cd $$d && $(MAKE) $@ ) || exit $$?; \ fi ; \ done (cd preloaded/src && $(MAKE) ../ebin/erts.app) -# ---------------------------------------------------------------------- -# These are "convenience targets", provided as shortcuts for developers -# - don't use them in scripts or assume they will always stay like this! -# - -.PHONY: $(FLAVORS) -$(FLAVORS): +.PHONY: smp +smp: $(V_at)for type in $(TYPES); do \ - ( $(MAKE) $$type FLAVOR=$@ ); \ + ( $(MAKE) $$type ) || exit $$?; \ done # Make erl script and erlc in $(ERL_TOP)/bin which runs the compiled version @@ -112,6 +107,11 @@ local_setup: $(ERL_TOP)/bin/start_clean.script \ $(ERL_TOP)/bin/no_dot_erlang.script +# ---------------------------------------------------------------------- +# These are "convenience targets", provided as shortcuts for developers +# - don't use them in scripts or assume they will always stay like this! +# + # Run the configure script .PHONY: configure configure: @@ -129,10 +129,8 @@ makefiles: .PHONY: release release: - $(V_at)for f in $(FLAVORS); do \ - for t in $(TYPES); do \ - ( cd emulator && $(MAKE) release FLAVOR=$$f TYPE=$$t ) \ - done \ + for t in $(TYPES); do \ + ( cd emulator && $(MAKE) release TYPE=$$t ) || exit $$?; \ done $(V_at)for d in $(ERTSDIRS) $(XINSTDIRS); do \ if test -d $$d; then \ diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 80bf236188..887babc13f 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -2726,6 +2726,21 @@ AC_DEFUN([LM_TRY_ENABLE_CFLAG], [ fi ]) +AC_DEFUN([LM_CHECK_ENABLE_CFLAG], [ + AC_MSG_CHECKING([whether $CC accepts $1...]) + saved_CFLAGS=$CFLAGS; + CFLAGS="$1 $CFLAGS"; + AC_TRY_COMPILE([],[return 0;],can_enable_flag=true,can_enable_flag=false) + CFLAGS=$saved_CFLAGS; + if test "X$can_enable_flag" = "Xtrue"; then + AS_VAR_SET($2, true) + AC_MSG_RESULT([yes]) + else + AS_VAR_SET($2, false) + AC_MSG_RESULT([no]) + fi +]) + dnl ERL_TRY_LINK_JAVA(CLASSES, FUNCTION-BODY dnl [ACTION_IF_FOUND [, ACTION-IF-NOT-FOUND]]) dnl Freely inspired by AC_TRY_LINK. (Maybe better to create a diff --git a/erts/configure.in b/erts/configure.in index 830e3d7776..9dec562f33 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -113,32 +113,14 @@ AS_HELP_STRING([--enable-bootstrap-only], # Disable stuff not necessary in a bootstrap only system in order # to speed up things by reducing the amount of stuff needing to be # built... - enable_threads=no - enable_smp_support=no with_termcap=no with_ssl=no with_ssl_zlib=no enable_hipe=no enable_sctp=no - enable_dirty_schedulers=no - fi + fi ]) -AC_ARG_ENABLE(threads, -AS_HELP_STRING([--enable-threads], [enable async thread support]) -AS_HELP_STRING([--disable-threads], [disable async thread support]), -[ case "$enableval" in - no) enable_threads=no ;; - *) enable_threads=yes ;; - esac ], enable_threads=unknown) - -AC_ARG_ENABLE(dirty-schedulers, -AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), -[ case "$enableval" in - no) enable_dirty_schedulers=no ;; - *) enable_dirty_schedulers=yes ;; - esac ], enable_dirty_schedulers=default) - AC_ARG_ENABLE(dirty-schedulers-test, AS_HELP_STRING([--enable-dirty-schedulers-test], [enable dirty scheduler test (for debugging purposes)]), [ case "$enableval" in @@ -146,22 +128,6 @@ AS_HELP_STRING([--enable-dirty-schedulers-test], [enable dirty scheduler test (f *) enable_dirty_schedulers_test=no ;; esac ], enable_dirty_schedulers_test=no) -AC_ARG_ENABLE(smp-support, -AS_HELP_STRING([--enable-smp-support], [enable smp support]) -AS_HELP_STRING([--disable-smp-support], [disable smp support]), -[ case "$enableval" in - no) enable_smp_support=no ;; - *) enable_smp_support=yes ;; - esac ], enable_smp_support=unknown) - -AC_ARG_ENABLE(plain-emulator, -AS_HELP_STRING([--enable-plain-emulator], [enable plain emulator]) -AS_HELP_STRING([--disable-plain-emulator], [disable plain emulator]), -[ case "$enableval" in - no) enable_plain_emulator=no ;; - *) enable_plain_emulator=yes ;; - esac ], enable_plain_emulator=unknown) - AC_ARG_ENABLE(smp-require-native-atomics, AS_HELP_STRING([--disable-smp-require-native-atomics], [disable the SMP requirement of a native atomic implementation]), @@ -580,6 +546,94 @@ AC_SUBST(WFLAGS) AC_SUBST(WERRORFLAGS) AC_SUBST(CFLAG_RUNTIME_LIBRARY_PATH) +## Check if we can do profile guided optimization of beam_emu +LM_CHECK_ENABLE_CFLAG([-fprofile-generate -Werror],[PROFILE_GENERATE]) +LM_CHECK_ENABLE_CFLAG([-fprofile-use -Werror],[PROFILE_USE]) + +## Check if this is clang +LM_CHECK_ENABLE_CFLAG([-fprofile-instr-generate -Werror],[PROFILE_INSTR_GENERATE]) +if test "X$PROFILE_INSTR_GENERATE" = "Xtrue"; then + # It was clang, now we also have to check if we have llvm-profdata and that + # we can link programs with -fprofile-instr-use + saved_CFLAGS=$CFLAGS; + CFLAGS="-fprofile-instr-generate -Werror $saved_CFLAGS" + AC_RUN_IFELSE([AC_LANG_PROGRAM([],[])], + [AC_CHECK_PROGS([LLVM_PROFDATA], [llvm-profdata]) + AC_CHECK_PROGS([XCRUN], [xcrun]) + if test "X$XCRUN" != "X" -a "X$LLVM_PROFDATA" = "X"; then + AC_MSG_CHECKING([for $XCRUN llvm-profdata]) + if $XCRUN llvm-profdata --help 2>& AS_MESSAGE_LOG_FD >& AS_MESSAGE_LOG_FD; then + LLVM_PROFDATA="$XCRUN llvm-profdata" + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + fi + AC_SUBST(LLVM_PROFDATA) + if test "X$LLVM_PROFDATA" != "X"; then + CFLAGS="-fprofile-instr-use=default.profdata -Werror $saved_CFLAGS"; + $LLVM_PROFDATA merge -output=default.profdata *.profraw; + AC_MSG_CHECKING([whether gcc accepts -fprofile-instr-use=default.profdata -Werror]) + AC_COMPILE_IFELSE([], + [AC_MSG_RESULT([yes]) + PROFILE_INSTR_USE=true], + [AC_MSG_RESULT([no]) + PROFILE_INSTR_USE=false]) + rm -f default.profdata + fi], + []) + rm -f *.profraw + CFLAGS=$saved_CFLAGS; +fi + +AC_ARG_ENABLE(pgo, +AS_HELP_STRING([--enable-pgo], + [build erts using PGO (profile guided optimization)]), +[ case "$enableval" in + no) enable_pgo=no ;; + *) enable_pgo=yes ;; + esac +],enable_pgo=default) + +USE_PGO=false +AC_MSG_CHECKING([whether to do PGO of erts]) +if test $enable_pgo = no; then + AC_MSG_RESULT([no, disabled by user]) +elif test $CROSS_COMPILING = yes; then + if $enable_pgo = yes; then + AC_MSG_ERROR(cannot use PGO when cross-compiling) + else + AC_MSG_RESULT([no, cross compiling]) + fi +elif test "X$host" = "Xwin32"; then + AC_MSG_RESULT([no, not supported in windows]) +elif test "X$PROFILE_GENERATE" = "Xtrue" -a "X$PROFILE_USE" = "Xtrue"; then + USE_PGO=true + AC_MSG_RESULT([yes, using -fprofile-generate]) + PROFILE_COMPILER=gcc +# check if $CC accepts -fprofile-correction, if so we can use PGO on multi-threaded files. + LM_CHECK_ENABLE_CFLAG([-fprofile-use -fprofile-correction -Werror],[PROFILE_CORRECTION]) + if test "X$PROFILE_CORRECTION" = "Xtrue"; then + PROFILE_CORRECTION="-fprofile-correction" + else + PROFILE_CORRECTION="" + fi + AC_SUBST(PROFILE_CORRECTION) +elif test "X$PROFILE_INSTR_GENERATE" = "Xtrue" -a "X$PROFILE_INSTR_USE" = "Xtrue"; then + USE_PGO=true + AC_MSG_RESULT([yes, using -fprofile-instr-generate]) + PROFILE_COMPILER=clang +else + if $enable_pgo = yes; then + AC_MSG_ERROR(cannot use PGO with this compiler) + else + AC_MSG_RESULT([no]) + fi +fi + +AC_SUBST(USE_PGO) +AC_SUBST(PROFILE_COMPILER) + AC_CHECK_SIZEOF(void *) # Needed for ARCH and smp checks below if test "x$ac_cv_sizeof_void_p" = x8; then AC_SUBST(EXTERNAL_WORD_SIZE, 64) @@ -640,6 +694,7 @@ case $chk_arch_ in armv7l) ARCH=arm;; armv7hl) ARCH=arm;; tile) ARCH=tile;; + e2k) ARCH=e2k;; *) ARCH=noarch;; esac @@ -996,81 +1051,12 @@ dnl are set by ERL_FIND_ETHR_LIB ERL_FIND_ETHR_LIB if test "X$ETHR_LIB_NAME" = "X"; then - found_threads=no -else - found_threads=yes + AC_MSG_ERROR([cannot build emulator since no thread library was found]) fi -FLAVORS= TYPES=opt -ERTS_BUILD_SMP_EMU=$enable_smp_support -AC_MSG_CHECKING(whether an emulator with smp support should be built) -case $ERTS_BUILD_SMP_EMU in - yes) - AC_MSG_RESULT(yes; enabled by user) - ;; - no) - AC_MSG_RESULT(no; disabled by user) - ;; - unknown) - AC_TRY_COMPILE([],[ - #if __GNUC__ >= 3 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 95) - ; - #else - #error old or no gcc - #endif - ], - gcc_smp=okgcc, - gcc_smp=oldornogcc) - ERTS_BUILD_SMP_EMU=yes - case "$enable_threads-$gcc_smp-$found_threads-$host_os" in - - no-*) - AC_MSG_RESULT(no; threads disabled by user) - ERTS_BUILD_SMP_EMU=no - ;; - - *-okgcc-yes-*) - AC_MSG_RESULT(yes) - ERTS_BUILD_SMP_EMU=yes - ;; - - *-win32) - AC_MSG_RESULT(yes) - ERTS_BUILD_SMP_EMU=yes - ;; - - *-oldornogcc-*) - AC_MSG_RESULT(no; old gcc or no gcc found) - ERTS_BUILD_SMP_EMU=no - ;; - - *) - AC_MSG_RESULT(no) - ERTS_BUILD_SMP_EMU=no - ;; - esac - ;; -esac - -AC_MSG_CHECKING(whether dirty schedulers should be enabled) -case $ERTS_BUILD_SMP_EMU-$enable_dirty_schedulers in - yes-yes) - DIRTY_SCHEDULER_SUPPORT=yes;; - yes-default) - DIRTY_SCHEDULER_SUPPORT=yes;; - no-default) - DIRTY_SCHEDULER_SUPPORT=no;; - no-yes) - AC_MSG_ERROR([No smp emulator will be built, but dirty schedulers requested]);; - *) - DIRTY_SCHEDULER_SUPPORT=no;; -esac -AC_MSG_RESULT($DIRTY_SCHEDULER_SUPPORT) -AC_SUBST(DIRTY_SCHEDULER_SUPPORT) DIRTY_SCHEDULER_TEST=$enable_dirty_schedulers_test -test $DIRTY_SCHEDULER_SUPPORT = yes || DIRTY_SCHEDULER_TEST=no AC_SUBST(DIRTY_SCHEDULER_TEST) test $DIRTY_SCHEDULER_TEST != yes || { test -f "$ERL_TOP/erts/CONF_INFO" || echo "" > "$ERL_TOP/erts/CONF_INFO" @@ -1085,26 +1071,15 @@ test $DIRTY_SCHEDULER_TEST != yes || { EOF } -if test $ERTS_BUILD_SMP_EMU = yes; then - - DEFAULT_FLAVOR=smp - FLAVORS="$FLAVORS smp" +test "X$smp_require_native_atomics" = "Xyes" && + AC_DEFINE(ETHR_SMP_REQUIRE_NATIVE_IMPLS, 1, [Define if you want to enable check for native ethread implementations]) - if test $found_threads = no; then - AC_MSG_ERROR([cannot build smp enabled emulator since no thread library was found]) - fi - - AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built]) - - test "X$smp_require_native_atomics" = "Xyes" && - AC_DEFINE(ETHR_SMP_REQUIRE_NATIVE_IMPLS, 1, [Define if you want to enable check for native ethread implementations]) - - case "$ethr_have_native_atomics-$smp_require_native_atomics-$ethr_have_native_spinlock" in - yes-*) - if test "$ethr_native_atomic_implementation" = "gcc_sync"; then - test -f "$ERL_TOP/erts/CONF_INFO" || - echo "" > "$ERL_TOP/erts/CONF_INFO" - cat >> $ERL_TOP/erts/CONF_INFO <<EOF +case "$ethr_have_native_atomics-$smp_require_native_atomics-$ethr_have_native_spinlock" in + yes-*) + if test "$ethr_native_atomic_implementation" = "gcc_sync"; then + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF WARNING: Only gcc's __sync_* builtins available for @@ -1121,18 +1096,18 @@ if test $ERTS_BUILD_SMP_EMU = yes; then more information. EOF - fi - ;; + fi + ;; - no-yes-*) - AC_MSG_ERROR([No native atomic implementation found. See the \"Atomic Memory Operations and the VM\" chapter of \$ERL_TOP/HOWTO/INSTALL.md for more information.]) - ;; + no-yes-*) + AC_MSG_ERROR([No native atomic implementation found. See the \"Atomic Memory Operations and the VM\" chapter of \$ERL_TOP/HOWTO/INSTALL.md for more information.]) + ;; - no-no-yes) + no-no-yes) - test -f "$ERL_TOP/erts/CONF_INFO" || - echo "" > "$ERL_TOP/erts/CONF_INFO" - cat >> $ERL_TOP/erts/CONF_INFO <<EOF + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF No native atomic implementation available. Fallbacks implemented using spinlocks will be @@ -1141,13 +1116,12 @@ EOF this. EOF - ;; - - no-no-no) + ;; - test -f "$ERL_TOP/erts/CONF_INFO" || - echo "" > "$ERL_TOP/erts/CONF_INFO" - cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF + no-no-no) + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF No native atomic implementation, nor no native spinlock implementation available. Fallbacks @@ -1156,76 +1130,11 @@ EOF will suffer immensely due to this. EOF - ;; - - esac - - enable_threads=force -fi - -AC_SUBST(ERTS_BUILD_SMP_EMU) - -ERTS_BUILD_PLAIN_EMU=$enable_plain_emulator -AC_MSG_CHECKING(whether an emulator without smp support should be built) -case $ERTS_BUILD_PLAIN_EMU in - yes) - AC_MSG_RESULT(yes; enabled by user) - ;; - no) - AC_MSG_RESULT(no; disabled by user) ;; - unknown) - case "$enable_threads-$ERTS_BUILD_SMP_EMU" in - no-*) - ERTS_BUILD_PLAIN_EMU=yes - AC_MSG_RESULT(yes) - ;; - *-no) - ERTS_BUILD_PLAIN_EMU=yes - AC_MSG_RESULT(yes; enabled as smp emulator was disabled) - ;; - *) - ERTS_BUILD_PLAIN_EMU=no - AC_MSG_RESULT(no) - ;; - esac - ;; -esac - -case $ERTS_BUILD_PLAIN_EMU in - yes) - AC_DEFINE(ERTS_HAVE_PLAIN_EMU, 1, [Define if the non-smp emulator is built]) - FLAVORS="$FLAVORS plain" - test -f "$ERL_TOP/erts/CONF_INFO" || echo "" > "$ERL_TOP/erts/CONF_INFO" - cat >> $ERL_TOP/erts/CONF_INFO <<EOF - The PLAIN aka NON-SMP emulator has been enabled. - This is a DEPRECATED feature scheduled for removal - in a future major release. - -EOF - ;; - no) - ;; esac - -AC_SUBST(ERTS_BUILD_PLAIN_EMU) -AC_SUBST(FLAVORS) AC_SUBST(TYPES) -case "$ERTS_BUILD_PLAIN_EMU-$ERTS_BUILD_SMP_EMU" in - no-no) - AC_MSG_ERROR([both smp and non-smp emulators have been disabled, one of them has to be enabled]) - ;; - *-no) - DEFAULT_FLAVOR=plain - ;; - *) - ;; -esac - -AC_SUBST(DEFAULT_FLAVOR) - AC_CHECK_FUNCS([posix_fadvise closefrom]) AC_CHECK_HEADERS([linux/falloc.h]) dnl * Old glibcs have broken fallocate64(). Make sure not to use it. @@ -1285,121 +1194,65 @@ if test $i_cv_posix_fallocate_works = yes; then fi # -# Figure out if the emulator should use threads. The default is set above -# in the enable_threads variable. It can have the following values: -# -# no single-threaded emulator requested -# yes multi-threaded emulator requested -# force multi-threaded emulator required -# # EMU_THR_LIB_NAME, EMU_THR_LIBS, EMU_THR_X_LIBS, and EMU_THR_DEFS is # used by the emulator, and can (but should not) be used by applications # that only require thread support when the emulator has thread support. # Other applications should use ETHR_LIB_NAME, ETHR_LIBS, ETHR_X_LIBS, # and ETHR_DEFS. # -AC_MSG_CHECKING(whether the emulator should use threads) EMU_THR_LIB_NAME= EMU_THR_X_LIBS= EMU_THR_LIBS= EMU_THR_DEFS= -emu_threads=no - -case "$enable_threads"-"$host_os" in - *-win32) - # The windows erlang emulator can never run without threads. - # It has to be enabled or the emulator will crash. Until that - # is fixed we force threads on win32. - enable_threads=force ;; - yes-osf*) - # The emulator hang when threads are enabled on osf - AC_MSG_ERROR(unresolved problems exist with threads on this platform) ;; - *) ;; -esac -case "$enable_threads"-"$found_threads" in - force-yes) - emu_threads=yes - AC_MSG_RESULT(yes; thread support required and therefore forced) ;; - yes-yes) - emu_threads=yes - AC_MSG_RESULT(yes; enabled by user) ;; - unknown-yes) - case $host_os in - solaris*|linux*|darwin*|win32) - emu_threads=yes - AC_MSG_RESULT(yes; default on this platform) - ;; - *) - AC_MSG_RESULT(no; default on this platform) - ;; - esac - ;; - no-yes) - AC_MSG_RESULT(no; thread support found but disabled by user) ;; - unknown-no|no-no) - AC_MSG_RESULT(no) ;; - force-no) - AC_MSG_ERROR(thread support required but not found) ;; - yes-no) - AC_MSG_ERROR(thread support enabled by user but not found) ;; - *) - AC_MSG_ERROR(internal error) ;; -esac +# Threads enabled for emulator +EMU_THR_LIB_NAME=$ETHR_LIB_NAME +EMU_THR_X_LIBS=$ETHR_X_LIBS +EMU_THR_LIBS=$ETHR_LIBS +EMU_THR_DEFS=$ETHR_DEFS +ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS threads" +AC_MSG_CHECKING(whether lock checking should be enabled) +AC_MSG_RESULT($enable_lock_check) +if test "x$enable_lock_check" != "xno"; then + EMU_THR_DEFS="$EMU_THR_DEFS -DERTS_ENABLE_LOCK_CHECK" +fi -if test $emu_threads != yes; then - enable_lock_check=no - enable_lock_count=no -else - # Threads enabled for emulator - EMU_THR_LIB_NAME=$ETHR_LIB_NAME - EMU_THR_X_LIBS=$ETHR_X_LIBS - EMU_THR_LIBS=$ETHR_LIBS - EMU_THR_DEFS=$ETHR_DEFS - ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS threads" - AC_MSG_CHECKING(whether lock checking should be enabled) - AC_MSG_RESULT($enable_lock_check) - if test "x$enable_lock_check" != "xno"; then - EMU_THR_DEFS="$EMU_THR_DEFS -DERTS_ENABLE_LOCK_CHECK" - fi +AC_MSG_CHECKING(whether lock counters should be enabled) +AC_MSG_RESULT($enable_lock_count) +if test "x$enable_lock_count" != "xno"; then + TYPES="$TYPES lcnt" +fi - AC_MSG_CHECKING(whether lock counters should be enabled) - AC_MSG_RESULT($enable_lock_count) - if test "x$enable_lock_count" != "xno"; then - TYPES="$TYPES lcnt" +case $host_os in + linux*) + AC_MSG_CHECKING([whether dlopen() needs to be called before first call to dlerror()]) + if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then + AC_DEFINE(ERTS_NEED_DLOPEN_BEFORE_DLERROR,[1], + [Define if dlopen() needs to be called before first call to dlerror()]) + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) fi + ;; + *) + ;; +esac - case $host_os in - linux*) - AC_MSG_CHECKING([whether dlopen() needs to be called before first call to dlerror()]) - if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then - AC_DEFINE(ERTS_NEED_DLOPEN_BEFORE_DLERROR,[1], - [Define if dlopen() needs to be called before first call to dlerror()]) - AC_MSG_RESULT(yes) - else - AC_MSG_RESULT(no) - fi - ;; - *) - ;; - esac - - # Remove -D_WIN32_WINNT*, -DWINVER* and -D_GNU_SOURCE from EMU_THR_DEFS - # (defined in CFLAGS). Note that we want to keep these flags - # in ETHR_DEFS, but not in EMU_THR_DEFS. - new_emu_thr_defs= - for thr_def in $EMU_THR_DEFS; do - case $thr_def in - -D_GNU_SOURCE*|-D_WIN32_WINNT*|-DWINVER*) - ;; - *) - new_emu_thr_defs="$new_emu_thr_defs $thr_def" - ;; - esac - done - EMU_THR_DEFS=$new_emu_thr_defs -fi +# Remove -D_WIN32_WINNT*, -DWINVER* and -D_GNU_SOURCE from EMU_THR_DEFS +# (defined in CFLAGS). Note that we want to keep these flags +# in ETHR_DEFS, but not in EMU_THR_DEFS. +new_emu_thr_defs= +for thr_def in $EMU_THR_DEFS; do + case $thr_def in + -D_GNU_SOURCE*|-D_WIN32_WINNT*|-DWINVER*) + ;; + *) + new_emu_thr_defs="$new_emu_thr_defs $thr_def" + ;; + esac +done +EMU_THR_DEFS=$new_emu_thr_defs AC_SUBST(EMU_THR_LIB_NAME) AC_SUBST(EMU_THR_X_LIBS) @@ -1784,6 +1637,8 @@ AC_CHECK_HEADER(sys/resource.h, [#include <sys/resource.h>])], [],[]) +AC_CHECK_FUNCS([getrusage]) + dnl Check if we have kernel poll support have_kernel_poll=no AC_CHECK_HEADER(sys/event.h, have_kernel_poll=kqueue) @@ -2449,9 +2304,6 @@ extern char end; #elif defined(HAVE__END_SYMBOL) extern char _end; #endif -#ifndef USE_THREADS -#undef ETHR_PTHREADS -#endif #ifdef ETHR_PTHREADS # ifdef ETHR_HAVE_PTHREAD_H @@ -2685,10 +2537,6 @@ extern char _end; # error no 'end' nor '_end' #endif -#ifndef USE_THREADS -#undef ETHR_PTHREADS -#endif - #ifdef ETHR_PTHREADS # ifdef ETHR_HAVE_PTHREAD_H # include <pthread.h> @@ -3481,10 +3329,7 @@ esac fi fi - - - - +AC_SUBST(FPE) dnl @@ -3618,62 +3463,24 @@ case $poll_works-$host_os in esac # -# If kqueue() found, check that it can be selected or polled on... +# If kqueue() found # if test $have_kernel_poll = kqueue; then - if test $poll_works = yes; then - kqueue_with=poll - else - kqueue_with=select - fi - AC_MSG_CHECKING([whether kqueue() fd can be ${kqueue_with}()ed on]) - AC_TRY_RUN([ -#include <stdlib.h> -#include <sys/types.h> -#include <sys/event.h> -#include <sys/time.h> -#ifdef ERTS_USE_POLL -#include <poll.h> -#else -#include <unistd.h> -#endif -int main(void) { - int kq = kqueue(); - if (kq < 0) return 2; - { -#ifdef ERTS_USE_POLL - struct pollfd pfds = {kq, POLLIN, 0}; - if (poll(&pfds, 1, 0) < 0) return 1; -#else - struct timeval tv = {0, 0}; - fd_set set; FD_ZERO(&set); FD_SET(kq, &set); - if (select(kq+1, &set, NULL, NULL, &tv) < 0) return 1; -#endif - } - return 0; -} - ], - ok_kqueue=yes, - ok_kqueue=no, - [ - case X$erl_xcomp_kqueue in - X) ok_kqueue=cross;; - Xyes|Xno) ok_kqueue=$erl_xcomp_kqueue;; - *) AC_MSG_ERROR([Bad erl_xcomp_kqueue value: $erl_xcomp_kqueue]);; - esac - ]) - AC_MSG_RESULT($ok_kqueue); - case $ok_kqueue in - yes) - ;; - cross) - have_kernel_poll=no - AC_MSG_WARN([result no guessed because of cross compilation]);; - *) - have_kernel_poll=no;; - esac +## Some OS X kernel version seems to have bugs in them with regards to kqueue +## Disable kernel poll on those versions + AC_MSG_CHECKING([whether host os has known kqueue bugs]) + case $host_os in + # Any OS X version < 16 has known problems with using kqueue + # so we don't use it there. See erl_poll.c for details. + darwin[[0-9]].*|darwin1[[0-5]].*) + AC_MSG_RESULT([yes, disabling kernel poll]) + have_kernel_poll=no + ;; + *) + AC_MSG_RESULT([no]) + ;; + esac fi - # # If epoll() found, check that it is level triggered. # @@ -3700,6 +3507,7 @@ fi # AC_MSG_CHECKING(whether kernel poll support should be enabled) ERTS_ENABLE_KERNEL_POLL=no +ERTS_BUILD_FALLBACK_POLL=no case $enable_kernel_poll-$have_kernel_poll in no-*) AC_MSG_RESULT(no; disabled by user);; @@ -3710,11 +3518,16 @@ case $enable_kernel_poll-$have_kernel_poll in *) case $have_kernel_poll in epoll) - AC_DEFINE(HAVE_SYS_EPOLL_H, 1, [Define if you have the <sys/epoll.h> header file.]);; + AC_DEFINE(HAVE_SYS_EPOLL_H, 1, [Define if you have the <sys/epoll.h> header file.]) + ERTS_BUILD_FALLBACK_POLL=yes + ;; /dev/poll) - AC_DEFINE(HAVE_SYS_DEVPOLL_H, 1, [Define if you have <sys/devpoll.h> header file.]);; + AC_DEFINE(HAVE_SYS_DEVPOLL_H, 1, [Define if you have <sys/devpoll.h> header file.]) + ;; kqueue) - AC_DEFINE(HAVE_SYS_EVENT_H, 1, [Define if you have <sys/event.h> header file.]);; + AC_DEFINE(HAVE_SYS_EVENT_H, 1, [Define if you have <sys/event.h> header file.]) + ERTS_BUILD_FALLBACK_POLL=yes + ;; *) AC_MSG_ERROR(configure.in need to be updated);; esac @@ -3722,7 +3535,7 @@ case $enable_kernel_poll-$have_kernel_poll in AC_DEFINE(ERTS_ENABLE_KERNEL_POLL, 1, [Define if you have kernel poll and want to use it]) AC_MSG_RESULT([yes; $have_kernel_poll]);; esac -AC_SUBST(ERTS_ENABLE_KERNEL_POLL) +AC_SUBST(ERTS_BUILD_FALLBACK_POLL) AC_MSG_CHECKING([whether putenv() stores a copy of the key-value pair]) AC_TRY_RUN([ @@ -3855,7 +3668,7 @@ case $host_os in darwin*) # Mach-O linker: a shared lib and a loadable # object file is not the same thing. - DED_LDFLAGS="-bundle -flat_namespace -undefined suppress" + DED_LDFLAGS="-bundle -bundle_loader ${ERL_TOP}/bin/$host/beam.smp" case $ARCH in amd64) DED_LDFLAGS="-m64 $DED_LDFLAGS" @@ -4980,7 +4793,7 @@ AH_BOTTOM([ # endif #endif -#if defined(DEBUG) && defined(USE_THREADS) && !defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(DEBUG) && !defined(ERTS_ENABLE_LOCK_CHECK) #define ERTS_ENABLE_LOCK_CHECK 1 #endif ]) diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile index b96cbbce40..444adf4a6e 100644 --- a/erts/doc/src/Makefile +++ b/erts/doc/src/Makefile @@ -173,6 +173,8 @@ release_docs_spec: docs "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(ERL_TOP)/erts/example/time_compat.erl \ "$(RELSYSDIR)/doc/html" + $(INSTALL_DATA) $(ERL_TOP)/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl \ + "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" diff --git a/erts/doc/src/alt_dist.xml b/erts/doc/src/alt_dist.xml index be969a8267..d3731a5391 100644 --- a/erts/doc/src/alt_dist.xml +++ b/erts/doc/src/alt_dist.xml @@ -47,23 +47,30 @@ runs on. The reason the C code is not made portable, is simply readability.</p> - <note> - <p>This section was written a long time ago. Most of it is still - valid, but some things have changed since then. - Most notably is the driver interface. Some updates have been made - to the documentation of the driver presented here, - but more can be done and is planned for the future. - The reader is encouraged to read the - <seealso marker="erl_driver"><c>erl_driver</c></seealso> and - <seealso marker="driver_entry"><c>driver_entry</c></seealso> - documentation also.</p> - </note> - <section> <title>Introduction</title> <p>To implement a new carrier for the Erlang distribution, the main steps are as follows.</p> + <note><p> + As of ERTS version 10.0 support for distribution controller + processes has been introduced. That is, the traffic over a + distribution channel can be managed by a process instead of + only by a port. This makes it possible to implement large + parts of the logic in Erlang code, and you perhaps do not + even need a new driver for the protocol. One example could + be Erlang distribution over UDP using <c>gen_udp</c> (your + Erlang code will of course have to take care of retranspissions, + etc in this example). That is, depending on what you want + to do you perhaps do not need to implement a driver at all + and can then skip the driver related sections below. + The <c>gen_tcp_dist</c> example described in the + <seealso marker="#distribution_module">Distribution + Module</seealso> section utilize distribution controller + processes and can be worth having a look at if you want to + use distribution controller processes. + </p></note> + <section> <title>Writing an Erlang Driver</title> <p>First, the protocol must be available to the Erlang machine, which @@ -152,7 +159,712 @@ </section> <section> + <marker id="distribution_module"/> + <title>Distribution Module</title> + <p> + The distribution module expose an API that <c>net_kernel</c> call + in order to manage connections to other nodes. The module name + should have the suffix <c>_dist</c>. + </p> + <p> + The module needs to create some kind of listening entity (process + or port) and an acceptor process that accepts incoming connections + using the listening entity. For each connection, the module at least + needs to create one connection supervisor process, which also is + responsible for the handshake when setting up the connection, and + a distribution controller (process or port) responsible for + transport of data over the connection. The distribution controller + and the connection supervisor process should be linked together + so both of them are cleaned up when the connection is taken down. + </p> + <p> + Note that there need to be exactly one distribution controller + per connection. A process or port can only be distribution + controller for one connection. The registration as distribution + controller cannot be undone. It will stick until the distribution + controller terminates. The distribution controller should not + ignore exit signals. It is allowed to trap exits, but it should + then voluntarily terminate when an exit signal is received. + </p> + <p> + An example implementation of a distribution module can be found + in + <url href="gen_tcp_dist.erl">$ERL_TOP/lib/kernel/examples/gen_tcp_dist/src/gen_tcp_dist.erl</url>. + It implements the distribution over TCP/IP using the <c>gen_tcp</c> + API with distribution controllers implemented by processes. This + instead of using port distribution controllers as the ordinary TCP/IP + distribution uses. + </p> + + <section> + <marker id="distribution_module_exported_callback_functions"/> + <title>Exported Callback Functions</title> + + <p> + The following functions are mandatory: + </p> + <taglist> + <tag><marker id="listen"/><c>listen(Name) -></c><br/> <c>{ok, {Listen, Address, Creation}} | {error, Error} </c></tag> + <item> + <p> + <c>listen/1</c> is called once in order to listen for incoming + connection requests. The call is made when the distribution is brought + up. The argument <c>Name</c> is the part of the node name before + the <c>@</c> sign in the full node name. It can be either an atom or a + string. + </p> + <p> + The return value consists of a <c>Listen</c> handle (which is + later passed to the <seealso marker="#accept"><c>accept/1</c></seealso> + callback), <c>Address</c> which is a <c>#net_address{}</c> record + with information about the address for the node (the + <c>#net_address{}</c> record is defined in + <c>kernel/include/net_address.hrl</c>), and <c>Creation</c> which + (currently) is an integer <c>1</c>, <c>2</c>, or <c>3</c>. + </p> + <p> + If <seealso marker="erts:epmd"><c>epmd</c></seealso> is to be used + for node discovery, you typically want to use the (unfortunately + undocumented) <c>erl_epmd</c> module (part of the <c>kernel</c> + application) in order to register the listen port with <c>epmd</c> + and retrieve <c>Creation</c> to use. + </p> + </item> + + <tag><marker id="accept"/><c>accept(Listen) -></c><br/> <c>AcceptorPid</c></tag> + <item> + <p> + <c>accept/1</c> should spawn a process that accepts connections. This + process should preferably execute on <c>max</c> priority. The process + identifier of this process should be returned. + </p> + <p> + The <c>Listen</c> argument will be the same as the <c>Listen</c> handle + part of the return value of the + <seealso marker="#listen"><c>listen/1</c></seealso> callback above. + <c>accept/1</c> is called only once when the distribution protocol is + started. + </p> + <p> + The caller of this function is a representative for <c>net_kernel</c> + (this may or may not be the process registered as <c>net_kernel</c>) + and is in this document identified as <c>Kernel</c>. + When a connection has been accepted by the acceptor process, it needs + to inform <c>Kernel</c> about the accepted connection. This is done by + passing a message on the form: + </p> + <code type="none"><![CDATA[Kernel ! {accept, AcceptorPid, DistController, Family, Proto}]]></code> + <p> + <c>DistController</c> is either the process or port identifier + of the distribution controller for the connection. The + distribution controller should be created by the acceptor + processes when a new connection is accepted. Its job is to + dispatch traffic on the connection. + </p> + <c>Kernel</c> responds with one of the following messages: + <taglist> + <tag><c>{Kernel, controller, SupervisorPid}</c></tag> + <item> + <p> + The request was accepted and <c>SupervisorPid</c> is the + process identifier of the connection supervisor process + (which is created in the + <seealso marker="#accept_connection"><c>accept_connection/5</c></seealso> + callback). + </p> + </item> + <tag><c>{Kernel, unsupported_protocol}</c></tag> + <item> + <p> + The request was rejected. This is a fatal error. The acceptor + process should terminate. + </p> + </item> + </taglist> + <p> + When an accept sequence has been completed the acceptor process + is expected to continue accepting further requests. + </p> + </item> + + <tag><marker id="accept_connection"/><c>accept_connection(AcceptorPid, DistCtrl, MyNode, Allowed, SetupTime) -></c><br/> <c>ConnectionSupervisorPid</c></tag> + <item> + <p> + <c>accept_connection/5</c> should spawn a process that will + perform the Erlang distribution handshake for the connection. + If the handshake successfully completes it should continue to + function as a connection supervisor. This process + should preferably execute on <c>max</c> priority. + </p> + <p>The arguments:</p> + <taglist> + <tag><c>AcceptorPid</c></tag> + <item> + <p> + Process identifier of the process created by the + <seealso marker="#accept"><c>accept/1</c></seealso> + callback. + </p> + </item> + <tag><c>DistCtrl</c></tag> + <item> + <p>The identifier of the distribution controller identifier + created by the acceptor process. To be passed along to + <c>dist_util:handshake_other_started(HsData)</c>. + </p> + </item> + <tag><c>MyNode</c></tag> + <item> + <p> + Node name of this node. To be passed along to + <c>dist_util:handshake_other_started(HsData)</c>. + </p> + </item> + <tag><c>Allowed</c></tag> + <item> + <p> + To be passed along to + <c>dist_util:handshake_other_started(HsData)</c>. + </p> + </item> + <tag><c>SetupTime</c></tag> + <item> + <p> + Time used for creating a setup timer by a + call to <c>dist_util:start_timer(SetupTime)</c>. + The timer should be passed along to + <c>dist_util:handshake_other_started(HsData)</c>. + </p> + </item> + </taglist> + <p> + The created process should provide callbacks and other + information needed for the handshake in a + <seealso marker="#hs_data_record"><c>#hs_data{}</c></seealso> + record and call <c>dist_util:handshake_other_started(HsData)</c> + with this record. + </p> + <p> + <c>dist_util:handshake_other_started(HsData)</c> will perform + the handshake and if the handshake successfully completes this + process will then continue in a connection supervisor loop + as long as the connection is up. + </p> + </item> + + <tag><marker id="setup"/><c>setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -></c><br/> <c>ConnectionSupervisorPid</c></tag> + <item> + <p> + <c>setup/5</c> should spawn a process that connects to + <c>Node</c>. When connection has been established it should + perform the Erlang distribution handshake for the connection. + If the handshake successfully completes it should continue to + function as a connection supervisor. This process + should preferably execute on <c>max</c> priority. + </p> + <p>The arguments:</p> + <taglist> + <tag><c>Node</c></tag> + <item> + <p> + Node name of remote node. To be passed along to + <c>dist_util:handshake_we_started(HsData)</c>. + </p> + </item> + <tag><c>Type</c></tag> + <item> + <p> + Connection type. To be passed along to + <c>dist_util:handshake_we_started(HsData)</c>. + </p> + </item> + <tag><c>MyNode</c></tag> + <item> + <p> + Node name of this node. To be passed along to + <c>dist_util:handshake_we_started(HsData)</c>. + </p> + </item> + <tag><c>LongOrShortNames</c></tag> + <item> + <p> + Either the atom <c>longnames</c> or + the atom <c>shortnames</c> indicating + whether long or short names is used. + </p> + </item> + <tag><c>SetupTime</c></tag> + <item> + <p> + Time used for creating a setup timer by a + call to <c>dist_util:start_timer(SetupTime)</c>. + The timer should be passed along to + <c>dist_util:handshake_we_started(HsData)</c>. + </p> + </item> + </taglist> + <p> + The caller of this function is a representative for <c>net_kernel</c> + (this may or may not be the process registered as <c>net_kernel</c>) + and is in this document identified as <c>Kernel</c>. + </p> + <p> + This function should, besides spawning the connection supervisor, + also create a distribution controller. The distribution + controller is either a process or a port which is responsible + for dispatching traffic. + </p> + <p> + The created process should provide callbacks and other + information needed for the handshake in a + <seealso marker="#hs_data_record"><c>#hs_data{}</c></seealso> + record and call <c>dist_util:handshake_we_started(HsData)</c> + with this record. + </p> + <p> + <c>dist_util:handshake_we_started(HsData)</c> will perform + the handshake and the handshake successfully completes this + process will then continue in a connection supervisor loop + as long as the connection is up. + </p> + </item> + + <tag><marker id="close"/><c>close(Listen) -></c><br/> <c>void()</c></tag> + + <item><p> + Called in order to close the <c>Listen</c> handle + that originally was passed from the + <seealso marker="#listen"><c>listen/1</c></seealso> callback. + </p></item> + + <tag><marker id="select"/><c>select(NodeName) -></c><br/> <c>boolean()</c></tag> + <item> + <p>Return <c>true</c> if the host name part + of the <c>NodeName</c> is valid for use + with this protocol; otherwise, <c>false</c>. + </p> + </item> + + </taglist> + + <p> + There are also two optional functions that may be + exported: + </p> + <taglist> + <tag><marker id="select"/><c>setopts(Listen, Opts) -></c><br/> <c>ok | {error, Error}</c></tag> + <item> + <p> + The argument <c>Listen</c> is the handle originally passed + from the + <seealso marker="#listen"><c>listen/1</c></seealso> callback. + The argument <c>Opts</c> is a list of options to set on future + connections. + </p> + </item> + + <tag><marker id="select"/><c>getopts(Listen, Opts) -></c><br/> <c>{ok, OptionValues} | {error, Error}</c></tag> + <item> + <p> + The argument <c>Listen</c> is the handle originally passed + from the + <seealso marker="#listen"><c>listen/1</c></seealso> callback. + The argument <c>Opts</c> is a list of options to read for future + connections. + </p> + </item> + </taglist> + + </section> + <section> + <marker id="hs_data_record"/> + <title>The #hs_data{} Record</title> + <p> + The <c>dist_util:handshake_we_started/1</c> and + <c>dist_util:handshake_other_started/1</c> functions + takes a <c>#hs_data{}</c> record as argument. There + are quite a lot of fields in this record that you + need to set. The record is defined in + <c>kernel/include/dist_util.hrl</c>. Not documented + fields should not be set, i.e., should be left as + <c>undefined</c>. + </p> + <p> + The following <c>#hs_data{}</c> record fields need + to be set unless otherwise stated:</p> + <taglist> + <tag><marker id="hs_data_kernel_pid"/><c>kernel_pid</c></tag> + <item> + <p> + Process identifier of the <c>Kernel</c> process. That is, + the process that called either + <seealso marker="#setup"><c>setup/5</c></seealso> or + <seealso marker="#accept_connection"><c>accept_connection/5</c></seealso>. + </p> + </item> + + <tag><marker id="hs_data_other_node"/><c>other_node</c></tag> + <item> + <p>Name of the other node. This field is only + mandatory when this node initiates the connection. + That is, when connection is set up via + <seealso marker="#setup"><c>setup/5</c></seealso>. + </p> + </item> + + <tag><marker id="hs_data_this_node"/><c>this_node</c></tag> + <item> + <p> + The node name of this node. + </p> + </item> + + <tag><marker id="hs_data_socket"/><c>socket</c></tag> + <item> + <p> + The identifier of the distribution controller. + </p> + </item> + + <tag><marker id="hs_data_timer"/><c>timer</c></tag> + <item> + <p> + The timer created using <c>dist_util:start_timer/1</c>. + </p> + </item> + + <tag><marker id="hs_data_allowed"/><c>allowed</c></tag> + <item> + <p>Information passed as <c>Allowed</c> to + <c>accept_connection/5</c>. This field is only + mandatory when the remote node initiated the + connection. That is, when the connection is set + up via + <seealso marker="#accept_connection"><c>accept_connection/5</c></seealso>. + </p> + </item> + + <tag><marker id="hs_data_f_send"/><c>f_send</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr, Data) -> ok | {error, Error}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of + the distribution controller and <c>Data</c> + is io data to pass to the other side. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_f_recv"/><c>f_recv</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr, Length) -> {ok, Packet} | {error, Reason}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of the distribution + controller. + If <c>Length</c> is <c>0</c>, all available bytes should be + returned. If <c>Length > 0</c>, exactly <c>Length</c> bytes + should be returned, or an error; possibly discarding less + than <c>Length</c> bytes of data when the connection is + closed from the other side. + It is used for passive receive of data from the + other end. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_f_setopts_pre_nodeup"/><c>f_setopts_pre_nodeup</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr) -> ok | {error, Error}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of + the distribution controller. Called just + before the distribution channel is taken up + for normal traffic. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_f_setopts_post_nodeup"/><c>f_setopts_post_nodeup</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr) -> ok | {error, Error}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of + the distribution controller. Called just + after distribution channel has been taken + up for normal traffic. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_f_getll"/><c>f_getll</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr) -> ID]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of + the distribution controller and <c>ID</c> is + the identifier of the low level entity that + handles the connection (often <c>DistCtrlr</c> + itself). + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_f_address"/><c>f_address</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr, Node) -> NetAddress]]></code> + <p> + where <c>DistCtrlr</c> is the identifier of + the distribution controller, <c>Node</c> + is the node name of the node on the other end, + and <c>NetAddress</c> is a <c>#net_address{}</c> + record with information about the address + for the <c>Node</c> on the other end of the + connection. The <c>#net_address{}</c> record + is defined in + <c>kernel/include/net_address.hrl</c>. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_mf_tick"/><c>mf_tick</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr) -> void()]]></code> + <p> + where <c>DistCtrlr</c> is the identifier + of the distribution controller. This + function should send information over + the connection that is not interpreted + by the other end while increasing the + statistics of received packets on the + other end. This is usually implemented by + sending an empty packet. + </p> + <note><p> + It is of vital importance that this operation + does not block the caller for a long time. + This since it is called from the connection + supervisor. + </p></note> + <p>Used when connection is up.</p> + </item> + + <tag><marker id="hs_data_mf_getstat"/><c>mf_getstat</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr) -> {ok, Received, Sent, PendSend}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier + of the distribution controller, <c>Received</c> + is received packets, <c>Sent</c> is + sent packets, and <c>PendSend</c> is + amount of packets in queue to be sent + or a <c>boolean()</c> indicating whether + there are packets in queue to be sent. + </p> + <note><p> + It is of vital importance that this operation + does not block the caller for a long time. + This since it is called from the connection + supervisor. + </p></note> + <p>Used when connection is up.</p> + </item> + + <tag><marker id="hs_data_request_type"/><c>request_type</c></tag> + <item> + <p> + The request <c>Type</c> as passed to + <seealso marker="#setup"><c>setup/5</c></seealso>. + This is only mandatory when the connection has + been initiated by this node. That is, the connection + is set up via <c>setup/5</c>. + </p> + </item> + + <tag><marker id="hs_data_mf_setopts"/><c>mf_setopts</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrl, Opts) -> ok | {error, Error}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier + of the distribution controller and <c>Opts</c> + is a list of options to set on the connection. + </p> + <p>This function is optional. Used when connection is up.</p> + </item> + + <tag><marker id="hs_data_mf_getopts"/><c>mf_getopts</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrl, Opts) -> {ok, OptionValues} | {error, Error}]]></code> + <p> + where <c>DistCtrlr</c> is the identifier + of the distribution controller and <c>Opts</c> + is a list of options to read for the connection. + </p> + <p>This function is optional. Used when connection is up.</p> + </item> + + <tag><marker id="hs_data_f_handshake_complete"/><c>f_handshake_complete</c></tag> + <item> + <p> + A fun with the following signature: + </p> + <code type="none"><![CDATA[fun (DistCtrlr, Node, DHandle) -> void()]]></code> + <p> + where <c>DistCtrlr</c> is the identifier + of the distribution controller, <c>Node</c> is + the node name of the node connected at the other + end, and <c>DHandle</c> is a distribution handle + needed by a distribution controller process when + calling the following BIFs: + </p> + <list> + <item><p><seealso marker="erts:erlang#dist_ctrl_get_data/1"><c>erlang:dist_ctrl_get_data/1</c></seealso></p></item> + <item><p><seealso marker="erts:erlang#dist_ctrl_get_data_notification/1"><c>erlang:dist_ctrl_get_data_notification/1</c></seealso></p></item> + <item><p><seealso marker="erts:erlang#dist_ctrl_input_handler/2"><c>erlang:dist_ctrl_input_handler/2</c></seealso></p></item> + <item><p><seealso marker="erts:erlang#dist_ctrl_put_data/2"><c>erlang:dist_ctrl_put_data/2</c></seealso></p></item> + </list> + <p> + This function is called when the handshake has + completed and the distribution channel is up. + The distribution controller can begin dispatching + traffic over the channel. This function is optional. + </p> + <p>Only used during handshake phase.</p> + </item> + + <tag><marker id="hs_data_add_flags"/><c>add_flags</c></tag> + <item> + <p> + <seealso marker="erl_dist_protocol#dflags">Distribution flags</seealso> + to add to the connection. Currently all (non obsolete) flags will + automatically be enabled. + </p> + <p> + This flag field is optional. + </p> + </item> + + <tag><marker id="hs_data_reject_flags"/><c>reject_flags</c></tag> + <item> + <p> + <seealso marker="erl_dist_protocol#dflags">Distribution flags</seealso> + to reject. Currently the following distribution flags can be rejected: + </p> + <taglist> + <tag><c>DFLAG_DIST_HDR_ATOM_CACHE</c></tag> + <item>Do not use atom cache over this connection.</item> + <tag><c>DFLAGS_STRICT_ORDER_DELIVERY</c></tag> + <item>Do not use any features that require strict + order delivery.</item> + </taglist> + <p> + This flag field is optional. + </p> + </item> + + <tag><marker id="hs_data_require_flags"/><c>require_flags</c></tag> + <item> + <p> + Require these <seealso marker="erl_dist_protocol#dflags">distribution + flags</seealso> to be used. The connection will be aborted during the + handshake if the other end does not use them. + </p> + <p> + This flag field is optional. + </p> + </item> + + </taglist> + </section> + + <section> + <marker id="distribution_data_delivery"/> + <title>Distribution Data Delivery</title> + <p> + When using the default configuration, the data to pass + over a connection needs to be delivered as is + to the node on the receiving end in the <em>exact same + order</em>, with no loss of data what so ever, as sent + from the sending node. + </p> + <p> + The data delivery order can be relaxed by disabling + features that require strict ordering. This is done by + passing the <c>?DFLAGS_STRICT_ORDER_DELIVERY</c> + <seealso marker="erl_dist_protocol#dflags">distribution + flags</seealso> in the + <seealso marker="alt_dist#hs_data_reject_flags"><c>reject_flags</c></seealso> + field of the <seealso marker="#hs_data_record"><c>#hs_data{}</c></seealso> + record used when setting up the connection. When relaxed + ordering is used, only the order of signals with the same + sender/receiver pair has to be preserved. + However, note that disabling the features that require + strict ordering may have a negative impact on performance, + throughput, and/or latency. + </p> + </section> + + <section> + <marker id="enable_your_distribution_module"/> + <title>Enable Your Distribution Module</title> + + <p>For <c>net_kernel</c> to find out which distribution module to use, + the <c>erl</c> command-line argument <c>-proto_dist</c> is used. It + is followed by one or more distribution module names, with suffix + "_dist" removed. That is, <c>gen_tcp_dist</c> as a distribution module + is specified as <c>-proto_dist gen_tcp</c>.</p> + + <p>If no <c>epmd</c> (TCP port mapper daemon) is used, also command-line + option <c>-no_epmd</c> is to be specified, which makes + Erlang skip the <c>epmd</c> startup, both as an OS process and as an + Erlang ditto.</p> + </section> + + </section> + + <section> <title>The Driver</title> + + <note> + <p>This section was written a long time ago. Most of it is still + valid, but some things have changed since then. Some updates have + been made to the documentation of the driver presented here, + but more can be done and is planned for the future. + The reader is encouraged to read the + <seealso marker="erl_driver"><c>erl_driver</c></seealso> and + <seealso marker="driver_entry"><c>driver_entry</c></seealso> + documentation also.</p> + </note> + <p>Although Erlang drivers in general can be beyond the scope of this section, a brief introduction seems to be in place.</p> diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml index 2421e0a8d9..e8c7e26457 100644 --- a/erts/doc/src/driver_entry.xml +++ b/erts/doc/src/driver_entry.xml @@ -196,10 +196,7 @@ typedef struct erl_drv_entry { char **rbuf, ErlDrvSizeT rlen, unsigned int *flags); /* Works mostly like 'control', a synchronous call into the driver */ - void (*event)(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); - /* Called when an event selected by - driver_event() has occurred */ + void* unused_event_callback; int extended_marker; /* ERL_DRV_EXTENDED_MARKER */ int major_version; /* ERL_DRV_EXTENDED_MAJOR_VERSION */ int minor_version; /* ERL_DRV_EXTENDED_MINOR_VERSION */ diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 638e88ca31..6f6eca2a0b 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -538,20 +538,6 @@ <p>Note that a distributed node will fail to start if epmd is not running.</p> </item> - <tag><marker id="smp"/><c><![CDATA[-smp [enable|auto|disable]]]></c></tag> - <item> - <p><c>-smp enable</c> and <c>-smp</c> start the Erlang runtime - system with SMP support enabled. This can fail if no runtime - system with SMP support is available. <c>-smp auto</c> starts - the Erlang runtime system with SMP support enabled if it is - available and more than one logical processor is detected. - <c>-smp disable</c> starts a runtime system without SMP support. - The runtime system without SMP support is deprecated and will - be removed in a future major release.</p> - <note> - <p>See also flag<seealso marker="#+S"><c>+S</c></seealso>.</p> - </note> - </item> <tag><c><![CDATA[-version]]></c> (emulator flag)</tag> <item> <p>Makes the emulator print its version number. The same @@ -776,13 +762,40 @@ <seealso marker="erlang#process_flag_message_queue_data"> <c>process_flag(message_queue_data, MQD)</c></seealso>.</p> </item> - <tag><c><![CDATA[+K true | false]]></c></tag> + <tag><marker id="+IOp"/><c>+IOp PollSets</c></tag> <item> - <p>Enables or disables the kernel poll functionality if supported by - the emulator. Defaults to <c><![CDATA[false]]></c> (disabled). - If the emulator does not support kernel poll, and flag - <c><![CDATA[+K]]></c> is passed to the emulator, a warning is - issued at startup.</p> + <p>Sets the number of IO pollsets to use when polling for I/O. + This option is only used on platforms that support concurrent + updates of a pollset, otherwise the same number of pollsets + are used as IO poll threads. + The default is 1. + </p> + </item> + <tag><marker id="+IOt"/><c>+IOt PollThreads</c></tag> + <item> + <p>Sets the number of IO poll threads to use when polling for I/O. + The maximum number of poll threads allowed is 1024. The default is 1. + </p> + <p>A good way to check if more IO poll threads are needed is to use + <seealso marker="runtime_tools:msacc">microstate accounting</seealso> + and see what the load of the IO poll thread is. If it is high it could + be a good idea to add more threads.</p> + </item> + <tag><marker id="+IOPp"/><c>+IOPp PollSetsPercentage</c></tag> + <item> + <p>Similar to <seealso marker="#+IOp"><c>+IOp</c></seealso> but uses + percentages to set the number of IO pollsets to create, based on the + number of poll threads configured. If both <c>+IOPp</c> and <c>+IOp</c> + are used, <c>+IOPp</c> is ignored. + </p> + </item> + <tag><marker id="+IOPt"/><c>+IOPt PollThreadsPercentage</c></tag> + <item> + <p>Similar to <seealso marker="#+IOt"><c>+IOt</c></seealso> but uses + percentages to set the number of IO poll threads to create, based on + the number of schedulers configured. If both <c>+IOPt</c> and + <c>+IOt</c> are used, <c>+IOPt</c> is ignored. + </p> </item> <tag><c><![CDATA[+l]]></c></tag> <item> @@ -902,7 +915,7 @@ <c><![CDATA[+S Schedulers:SchedulerOnline]]></c></tag> <item> <p>Sets the number of scheduler threads to create and scheduler threads - to set online when SMP support has been enabled. The maximum for both + to set online. The maximum for both values is 1024. If the Erlang runtime system is able to determine the number of logical processors configured and logical processors available, <c>Schedulers</c> defaults to logical processors @@ -920,8 +933,6 @@ <p>Specifying value <c>0</c> for <c>Schedulers</c> or <c>SchedulersOnline</c> resets the number of scheduler threads or scheduler threads online, respectively, to its default value.</p> - <p>This option is ignored if the emulator does not have SMP support - enabled (see flag <seealso marker="#smp"><c>-smp</c></seealso>).</p> </item> <tag><marker id="+SP"/><c><![CDATA[+SP SchedulersPercentage:SchedulersOnlinePercentage]]></c></tag> @@ -929,8 +940,8 @@ <p>Similar to <seealso marker="#+S"><c>+S</c></seealso> but uses percentages to set the number of scheduler threads to create, based on logical processors configured, and scheduler threads to set online, - based on logical processors available, when SMP support has been - enabled. Specified values must be > 0. For example, + based on logical processors available. + Specified values must be > 0. For example, <c>+SP 50:25</c> sets the number of scheduler threads to 50% of the logical processors configured, and the number of scheduler threads online to 25% of the logical processors available. @@ -945,15 +956,13 @@ and 8 logical cores available, the combination of the options <c>+S 4:4 +SP 50:25</c> (in either order) results in 2 scheduler threads (50% of 4) and 1 scheduler thread online (25% of 4).</p> - <p>This option is ignored if the emulator does not have SMP support - enabled (see flag <seealso marker="#smp"><c>-smp</c></seealso>).</p> </item> <tag><marker id="+SDcpu"/><c><![CDATA[+SDcpu DirtyCPUSchedulers:DirtyCPUSchedulersOnline]]></c></tag> <item> <p>Sets the number of dirty CPU scheduler threads to create and dirty - CPU scheduler threads to set online when threading support has been - enabled. The maximum for both values is 1024, and each value is + CPU scheduler threads to set online. + The maximum for both values is 1024, and each value is further limited by the settings for normal schedulers:</p> <list type="bulleted"> <item>The number of dirty CPU scheduler threads created cannot exceed @@ -977,16 +986,14 @@ executing on ordinary schedulers. If the amount of dirty CPU schedulers was allowed to be unlimited, dirty CPU bound jobs would potentially starve normal jobs.</p> - <p>This option is ignored if the emulator does not have threading - support enabled.</p> </item> <tag><marker id="+SDPcpu"/><c><![CDATA[+SDPcpu DirtyCPUSchedulersPercentage:DirtyCPUSchedulersOnlinePercentage]]></c></tag> <item> <p>Similar to <seealso marker="#+SDcpu"><c>+SDcpu</c></seealso> but uses percentages to set the number of dirty CPU scheduler threads to - create and the number of dirty CPU scheduler threads to set online - when threading support has been enabled. Specified values must be + create and the number of dirty CPU scheduler threads to set online. + Specified values must be > 0. For example, <c>+SDPcpu 50:25</c> sets the number of dirty CPU scheduler threads to 50% of the logical processors configured and the number of dirty CPU scheduler threads online to 25% of the @@ -1003,13 +1010,11 @@ the combination of the options <c>+SDcpu 4:4 +SDPcpu 50:25</c> (in either order) results in 2 dirty CPU scheduler threads (50% of 4) and 1 dirty CPU scheduler thread online (25% of 4).</p> - <p>This option is ignored if the emulator does not have threading - support enabled.</p> </item> <tag><marker id="+SDio"/><c><![CDATA[+SDio DirtyIOSchedulers]]></c></tag> <item> - <p>Sets the number of dirty I/O scheduler threads to create when - threading support has been enabled. Valid range is 0-1024. By + <p>Sets the number of dirty I/O scheduler threads to create. + Valid range is 0-1024. By default, the number of dirty I/O scheduler threads created is 10, same as the default number of threads in the <seealso marker="#async_thread_pool_size">async thread pool</seealso>.</p> @@ -1019,8 +1024,6 @@ expected to execute on dirty I/O schedulers. If the user should schedule CPU bound jobs on dirty I/O schedulers, these jobs might starve ordinary jobs executing on ordinary schedulers.</p> - <p>This option is ignored if the emulator does not have threading - support enabled.</p> </item> <tag><c><![CDATA[+sFlag Value]]></c></tag> <item> @@ -1297,25 +1300,6 @@ <seealso marker="erlang#system_info_cpu_topology"> <c>erlang:system_info(cpu_topology)</c></seealso>.</p> </item> - <tag><marker id="+secio"/><c>+secio true|false</c></tag> - <item> - <p>Enables or disables eager check I/O scheduling. Defaults - to <c>true</c>. The default was changed from <c>false</c> - as from ERTS 7.0. The behavior before this - flag was introduced corresponds to <c>+secio false</c>.</p> - <p>The flag effects when schedulers will check for I/O - operations possible to execute, and when such I/O operations - will execute. As the parameter name implies, - schedulers are more eager to check for I/O when - <c>true</c> is passed. This, however, also implies that - execution of outstanding I/O operation is not - prioritized to the same extent as when <c>false</c> is - passed.</p> - <p><seealso marker="erlang#system_info_eager_check_io"> - <c>erlang:system_info(eager_check_io)</c></seealso> - returns the value of this parameter used when starting - the virtual machine.</p> - </item> <tag><marker id="+sfwi"/><c>+sfwi Interval</c></tag> <item> <p>Sets scheduler-forced wakeup interval. All run queues are diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 610351db6c..a78b13aaa4 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -829,7 +829,31 @@ DiB == gen_digest(ChA, ICA)? <item> <p>The node understand UTF-8 encoded atoms.</p> </item> + <tag><c>-define(DFLAG_MAP_TAG, 16#20000).</c></tag> + <item> + <p>The node understand the map tag.</p> + </item> + <tag><c>-define(DFLAG_BIG_CREATION, 16#40000).</c></tag> + <item> + <p>The node understand big node creation.</p> + </item> + <tag><c>-define(DFLAG_SEND_SENDER, 16#80000).</c></tag> + <item> + <p> + Use the <c>SEND_SENDER</c> + <seealso marker="#control_message">control message</seealso> + instead of the <c>SEND</c> control message and use the + <c>SEND_SENDER_TT</c> control message instead + of the <c>SEND_TT</c> control message. + </p> + </item> </taglist> + <p> + There are also a collection of <c>DFLAG</c>s bitwise or:ed + together in the <c>DFLAGS_STRICT_ORDER_DELIVERY</c> macro. + These flags corresponds to features that require strict + ordering of data over distribution channels. + </p> </section> </section> @@ -922,6 +946,7 @@ DiB == gen_digest(ChA, ICA)? </item> </taglist> + <marker id="control_message"/> <p>The <c>ControlMessage</c> is a tuple, where the first element indicates which distributed operation it encodes:</p> @@ -1028,4 +1053,49 @@ DiB == gen_digest(ChA, ICA)? </item> </taglist> </section> + + <section> + <title>New Ctrlmessages for Erlang/OTP 21</title> + <taglist> + <tag><c>SEND_SENDER</c></tag> + <item> + <p><c>{22, FromPid, ToPid}</c></p> + <p>Followed by <c>Message</c>.</p> + <p> + This control messages replace the <c>SEND</c> control + message and will be sent when the distribution flag + <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_SEND_SENDER</c></seealso> + has been negotiated in the connection setup handshake. + </p> + <note><p> + Messages encoded before the connection has + been set up may still use the <c>SEND</c> control + message. However, once a <c>SEND_SENDER</c> or <c>SEND_SENDER_TT</c> + control message has been sent, no more <c>SEND</c> + control messages will be sent in the same direction + on the connection. + </p></note> + </item> + <tag><c>SEND_SENDER_TT</c></tag> + <item> + <p><c>{23, FromPid, ToPid, TraceToken}</c></p> + <p>Followed by <c>Message</c>.</p> + <p> + This control messages replace the <c>SEND_TT</c> control + message and will be sent when the distribution flag + <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_SEND_SENDER</c></seealso> + has been negotiated in the connection setup handshake. + </p> + <note><p> + Messages encoded before the connection has + been set up may still use the <c>SEND_TT</c> control + message. However, once a <c>SEND_SENDER</c> or <c>SEND_SENDER_TT</c> + control message has been sent, no more <c>SEND_TT</c> + control messages will be sent in the same direction + on the connection. + </p></note> + </item> + </taglist> + </section> + </chapter> diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 5705100ab2..ca9d458e1e 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -157,7 +157,7 @@ </note> <p>Most functions in this API are <em>not</em> thread-safe, that is, - they <em>cannot</em> be called from any thread. Functions + they <em>cannot</em> be called from arbitrary threads. Functions that are not documented as thread-safe can only be called from driver callbacks or function calls descending from a driver callback call. Notice that driver callbacks can be called from diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 5a69bed34c..419e41693e 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -206,7 +206,7 @@ ok <seealso marker="#enif_make_resource"> <c>enif_make_resource</c></seealso>. The term returned by <c>enif_make_resource</c> is opaque in nature. - It can be stored and passed between processes on the same node, but + It can be stored and passed between processes, but the only real end usage is to pass it back as an argument to a NIF. The NIF can then call <seealso marker="#enif_get_resource"> <c>enif_get_resource</c></seealso> and get back a pointer to the @@ -344,6 +344,81 @@ return term;</code> <c>enif_convert_time_unit()</c></seealso></item> </list> </item> + <tag><marker id="enif_ioq"/>I/O Queues</tag> + <item> + <p>The Erlang nif library contains function for easily working + with I/O vectors as used by the unix system call <c>writev</c>. + The I/O Queue is not thread safe, so some other synchronization + mechanism has to be used.</p> + <list type="bulleted"> + <item><seealso marker="#SysIOVec"> + <c>SysIOVec</c></seealso></item> + <item><seealso marker="#ErlNifIOVec"> + <c>ErlNifIOVec</c></seealso></item> + <item><seealso marker="#enif_ioq_create"> + <c>enif_ioq_create()</c></seealso></item> + <item><seealso marker="#enif_ioq_destroy"> + <c>enif_ioq_destroy()</c></seealso></item> + <item><seealso marker="#enif_ioq_enq_binary"> + <c>enif_ioq_enq_binary()</c></seealso></item> + <item><seealso marker="#enif_ioq_enqv"> + <c>enif_ioq_enqv()</c></seealso></item> + <item><seealso marker="#enif_ioq_deq"> + <c>enif_ioq_deq()</c></seealso></item> + <item><seealso marker="#enif_ioq_peek"> + <c>enif_ioq_peek()</c></seealso></item> + <item><seealso marker="#enif_inspect_iovec"> + <c>enif_inspect_iovec()</c></seealso></item> + <item><seealso marker="#enif_free_iovec"> + <c>enif_free_iovec()</c></seealso></item> + </list> + <p>Typical usage when writing to a file descriptor looks like this:</p> + <code type="none"><![CDATA[ +int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, + ErlNifIOQueue *q, int fd) { + + ErlNifIOVec vec, *iovec = &vec; + SysIOVec *sysiovec; + int saved_errno; + int iovcnt, n; + + if (!enif_inspect_iovec(env, 64, term, tail, &iovec)) + return -2; + + if (enif_ioq_size(q) > 0) { + /* If the I/O queue contains data we enqueue the iovec and + then peek the data to write out of the queue. */ + if (!enif_ioq_enqv(q, iovec, 0)) + return -3; + + sysiovec = enif_ioq_peek(q, &iovcnt); + } else { + /* If the I/O queue is empty we skip the trip through it. */ + iovcnt = iovec->iovcnt; + sysiovec = iovec->iov; + } + + /* Attempt to write the data */ + n = writev(fd, sysiovec, iovcnt); + saved_errno = errno; + + if (enif_ioq_size(q) == 0) { + /* If the I/O queue was initially empty we enqueue any + remaining data into the queue for writing later. */ + if (n >= 0 && !enif_ioq_enqv(q, iovec, n)) + return -3; + } else { + /* Dequeue any data that was written from the queue. */ + if (n > 0 && !enif_ioq_deq(q, n, NULL)) + return -4; + } + + /* return n, which is either number of bytes written or -1 if + some error happened */ + errno = saved_errno; + return n; +}]]></code> + </item> <tag><marker id="lengthy_work"/>Long-running NIFs</tag> <item> <p>As mentioned in the <seealso marker="#WARNING">warning</seealso> text @@ -837,6 +912,36 @@ typedef enum { </item> </taglist> </item> + <tag><marker id="SysIOVec"/><c>SysIOVec</c></tag> + <item> + <p>A system I/O vector, as used by <c>writev</c> on + Unix and <c>WSASend</c> on Win32. It is used in + <c>ErlNifIOVec</c> and by + <seealso marker="#enif_ioq_peek"><c>enif_ioq_peek</c></seealso>.</p> + </item> + <tag><marker id="ErlNifIOVec"/><c>ErlNifIOVec</c></tag> + <item> + <code type="none"> +typedef struct { + int iovcnt; + size_t size; + SysIOVec* iov; +} ErlNifIOVec;</code> + <p>An I/O vector containing <c>iovcnt</c> <c>SysIOVec</c>s + pointing to the data. It is used by + <seealso marker="#enif_inspect_iovec"> + <c>enif_inspect_iovec</c></seealso> and + <seealso marker="#enif_ioq_enqv"> + <c>enif_ioq_enqv</c></seealso>.</p> + </item> + <tag><marker id="ErlNifIOQueueOpts"/><c>ErlNifIOQueueOpts</c></tag> + <item> + Options to configure a <c>ErlNifIOQueue</c>. + <taglist> + <tag>ERL_NIF_IOQ_NORMAL</tag> + <item><p>Create a normal I/O Queue</p></item> + </taglist> + </item> </taglist> </section> @@ -1143,6 +1248,31 @@ typedef enum { </func> <func> + <name><ret>void</ret> + <nametext>enif_free_iovec(ErlNifIOvec* iov)</nametext></name> + <fsummary>Free an ErlIOVec</fsummary> + <desc> + <p>Frees an io vector returned from + <seealso marker="#enif_inspect_iovec"> + <c>enif_inspect_iovec</c></seealso>. + This is needed only if a <c>NULL</c> environment is passed to + <seealso marker="#enif_inspect_iovec"> + <c>enif_inspect_iovec</c></seealso>.</p> + <code type="none"><![CDATA[ +ErlNifIOVec *iovec = NULL; +size_t max_elements = 128; +ERL_NIF_TERM tail; +if (!enif_inspect_iovec(NULL, max_elements, term, &tail, iovec)) + return 0; + +// Do things with the iovec + +/* Free the iovector, possibly in another thread or nif function call */ +enif_free_iovec(iovec);]]></code> + </desc> + </func> + + <func> <name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, ERL_NIF_TERM term, char* buf, unsigned size, ErlNifCharEncoding encode)</nametext> </name> @@ -1449,6 +1579,127 @@ typedef enum { </func> <func> + <name><ret>int</ret><nametext>enif_inspect_iovec(ErlNifEnv* + env, size_t max_elements, ERL_NIF_TERM iovec_term, ERL_NIF_TERM* tail, + ErlNifIOVec** iovec)</nametext></name> + <fsummary>Inspect a list of binaries as an ErlNifIOVec.</fsummary> + <desc> + <p>Fills <c>iovec</c> with the list of binaries provided in + <c>iovec_term</c>. The number of elements handled in the call is + limited to <c>max_elements</c>, and <c>tail</c> is set to the + remainder of the list. Note that the output may be longer than + <c>max_elements</c> on some platforms. + </p> + <p>To create a list of binaries from an arbitrary iolist, use + <seealso marker="erts:erlang#iolist_to_iovec/1"> + <c>erlang:iolist_to_iovec/1</c></seealso>.</p> + <p>When calling this function, <c>iovec</c> should contain a pointer to + <c>NULL</c> or a ErlNifIOVec structure that should be used if + possible. e.g. + </p> + <code type="none"> +/* Don't use a pre-allocated structure */ +ErlNifIOVec *iovec = NULL; +enif_inspect_iovec(env, max_elements, term, &tail, &iovec); + +/* Use a stack-allocated vector as an optimization for vectors with few elements */ +ErlNifIOVec vec, *iovec = &vec; +enif_inspect_iovec(env, max_elements, term, &tail, &iovec); +</code> + <p>The contents of the <c>iovec</c> is valid until the called nif + function returns. If the <c>iovec</c> should be valid after the nif + call returns, it is possible to call this function with a + <c>NULL</c> environment. If no environment is given the <c>iovec</c> + owns the data in the vector and it has to be explicitly freed using + <seealso marker="#enif_free_iovec"><c>enif_free_iovec</c> + </seealso>.</p> + <p>Returns <c>true</c> on success, or <c>false</c> if <c>iovec_term</c> + not an iovec.</p> + </desc> + </func> + + <func> + <name><ret>ErlNifIOQueue *</ret> + <nametext>enif_ioq_create(ErlNifIOQueueOpts opts)</nametext></name> + <fsummary>Create a new IO Queue</fsummary> + <desc> + <p>Create a new I/O Queue that can be used to store data. + <c>opts</c> has to be set to <c>ERL_NIF_IOQ_NORMAL</c>. + </p> + </desc> + </func> + + <func> + <name><ret>void</ret> + <nametext>enif_ioq_destroy(ErlNifIOQueue *q)</nametext></name> + <fsummary>Destroy an IO Queue and free it's content</fsummary> + <desc> + <p>Destroy the I/O queue and free all of it's contents</p> + </desc> + </func> + + <func> + <name><ret>int</ret> + <nametext>enif_ioq_deq(ErlNifIOQueue *q, size_t count, size_t *size)</nametext></name> + <fsummary>Dequeue count bytes from the IO Queue</fsummary> + <desc> + <p>Dequeue <c>count</c> bytes from the I/O queue. + If <c>size</c> is not <c>NULL</c>, the new size of the queue + is placed there.</p> + <p>Returns <c>true</c> on success, or <c>false</c> if the I/O does + not contain <c>count</c> bytes. On failure the queue is left un-altered.</p> + </desc> + </func> + + <func> + <name><ret>int</ret> + <nametext>enif_ioq_enq_binary(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip)</nametext></name> + <fsummary>Enqueue the binary into the IO Queue</fsummary> + <desc> + <p>Enqueue the <c>bin</c> into <c>q</c> skipping the first <c>skip</c> bytes.</p> + <p>Returns <c>true</c> on success, or <c>false</c> if <c>skip</c> is greater + than the size of <c>bin</c>. Any ownership of the binary data is transferred + to the queue and <c>bin</c> is to be considered read-only for the rest of the NIF + call and then as released.</p> + </desc> + </func> + + <func> + <name><ret>int</ret> + <nametext>enif_ioq_enqv(ErlNifIOQueue *q, ErlNifIOVec *iovec, size_t skip)</nametext></name> + <fsummary>Enqueue the iovec into the IO Queue</fsummary> + <desc> + <p>Enqueue the <c>iovec</c> into <c>q</c> skipping the first <c>skip</c> bytes.</p> + <p>Returns <c>true</c> on success, or <c>false</c> if <c>skip</c> is greater + than the size of <c>iovec</c>.</p> + </desc> + </func> + + <func> + <name><ret>SysIOVec *</ret> + <nametext>enif_ioq_peek(ErlNifIOQueue *q, int *iovlen)</nametext></name> + <fsummary>Peek inside the IO Queue</fsummary> + <desc> + <p>Get the I/O queue as a pointer to an array of <c>SysIOVec</c>s. + It also returns the number of elements in <c>iovlen</c>. + This is the only way to get data out of the queue.</p> + <p>Nothing is removed from the queue by this function, that must be done + with <seealso marker="#enif_ioq_deq"><c>enif_ioq_deq</c></seealso>.</p> + <p>The returned array is suitable to use with the Unix system + call <c>writev</c>.</p> + </desc> + </func> + + <func> + <name><ret>size_t</ret> + <nametext>enif_ioq_size(ErlNifIOQueue *q)</nametext></name> + <fsummary>Get the current size of the IO Queue</fsummary> + <desc> + <p>Get the size of <c>q</c>.</p> + </desc> + </func> + + <func> <name><ret>int</ret> <nametext>enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term)</nametext> </name> @@ -1952,10 +2203,33 @@ typedef enum { details, see the <seealso marker="#enif_resource_example">example of creating and returning a resource object</seealso> in the User's Guide.</p> - <p>Notice that the only defined behavior of using a resource term in - an Erlang program is to store it and send it between processes on the - same node. Other operations, such as matching or - <c>term_to_binary</c>, have unpredictable (but harmless) results.</p> + <note> + <p>Since ERTS 9.0 (OTP-20.0), resource terms have a defined behavior + when compared and serialized through <c>term_to_binary</c> or passed + between nodes.</p> + <list type="bulleted"> + <item> + <p>Two resource terms will compare equal iff they + would yield the same resource object pointer when passed to + <seealso marker="#enif_get_resource"><c>enif_get_resource</c></seealso>.</p> + </item> + <item> + <p>A resoure term can be serialized with <c>term_to_binary</c> and later + be fully recreated if the resource object is still alive when + <c>binary_to_term</c> is called. A <em>stale</em> resource term will be + returned from <c>binary_to_term</c> if the resource object has + been deallocated. <seealso marker="#enif_get_resource"><c>enif_get_resource</c></seealso> + will return false for stale resource terms.</p> + <p>The same principles of serialization apply when passing + resource terms in messages to remote nodes and back again. A + resource term will act stale on all nodes except the node where + its resource object is still alive in memory.</p> + </item> + </list> + <p>Before ERTS 9.0 (OTP-20.0), all resource terms did + compare equal to each other and to empty binaries (<c><<>></c>). + If serialized, they would be recreated as plain empty binaries.</p> + </note> </desc> </func> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 687ff38cbf..8dbebe880d 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -60,6 +60,14 @@ </desc> </datatype> <datatype> + <name>iovec()</name> + <desc> + <p>A list of binaries. This datatype is useful to use + together with <seealso marker="erl_nif#enif_inspect_iovec"> + <c>enif_inspect_iovec</c></seealso>.</p> + </desc> + </datatype> + <datatype> <name name="message_queue_data"></name> <desc> <p>See <seealso marker="#process_flag_message_queue_data"> @@ -181,6 +189,14 @@ </taglist> </desc> </datatype> + + <datatype> + <name name="dist_handle"></name> + <desc> + <p>An opaque handle identifing a distribution channel.</p> + </desc> + </datatype> + </datatypes> <funcs> @@ -424,6 +440,16 @@ Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).</code> <seealso marker="#binary_to_atom/2"><c>binary_to_atom/2</c></seealso>, but the atom must exist.</p> <p>Failure: <c>badarg</c> if the atom does not exist.</p> + <note> + <p>Note that the compiler may optimize away atoms. For + example, the compiler will rewrite + <c>atom_to_list(some_atom)</c> to <c>"some_atom"</c>. If + that expression is the only mention of the atom + <c>some_atom</c> in the containing module, the atom will not + be created when the module is loaded, and a subsequent call + to <c>binary_to_existing_atom(<<"some_atom">>, utf8)</c> + will fail.</p> + </note> </desc> </func> @@ -1215,6 +1241,141 @@ end</code> </func> <func> + <name name="dist_ctrl_get_data" arity="1"/> + <fsummary>Get distribution channel data to pass to another node.</fsummary> + <desc> + <p> + Get distribution channel data from the local node that is + to be passed to the remote node. The distribution channel + is identified by <c><anno>DHandle</anno></c>. If no data + is available, the atom <c>none</c> is returned. One + can request to be informed by a message when more + data is available by calling + <seealso marker="erlang#dist_ctrl_get_data_notification/1"><c>erlang:dist_ctrl_get_data_notification(DHandle)</c></seealso>. + </p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> + <name name="dist_ctrl_get_data_notification" arity="1"/> + <fsummary>Request notification about available outgoing distribution channel data.</fsummary> + <desc> + <p> + Request notification when more data is available to + fetch using + <seealso marker="erlang#dist_ctrl_get_data/1"><c>erlang:dist_ctrl_get_data(DHandle)</c></seealso> + for the distribution channel identified by + <c><anno>DHandle</anno></c>. When more data is present, + the caller will be sent the message <c>dist_data</c>. + Once a <c>dist_data</c> messages has been sent, no + more <c>dist_data</c> messages will be sent until + the <c>dist_ctrl_get_data_notification/1</c> function has been called + again. + </p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> + <name name="dist_ctrl_input_handler" arity="2"/> + <fsummary>Register distribution channel input handler process.</fsummary> + <desc> + <p> + Register an alternate input handler process for the + distribution channel identified by <c><anno>DHandle</anno></c>. + Once this function has been called, <c><anno>InputHandler</anno></c> + is the only process allowed to call + <seealso marker="erlang#dist_ctrl_put_data/2"><c>erlang:dist_ctrl_put_data(DHandle, Data)</c></seealso> + with the <c><anno>DHandle</anno></c> identifing this distribution + channel. + </p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> + <name name="dist_ctrl_put_data" arity="2"/> + <fsummary>Pass data into the VM from a distribution channel.</fsummary> + <desc> + <p> + Deliver distribution channel data from a remote node to the + local node. + </p> + <note><p> + Only the process registered as distribution + controller for the distribution channel identified by + <c><anno>DHandle</anno></c> is allowed to call this + function unless an alternate input handler process + has been registered using + <seealso marker="erlang#dist_ctrl_input_handler/2"><c>erlang:dist_ctrl_input_handler(DHandle, InputHandler)</c></seealso>. + If an alternate input handler has been registered, only + the registered input handler process is allowed to call + this function. + </p></note> + <p> + This function is used when implementing an alternative + distribution carrier using processes as distribution + controllers. <c><anno>DHandle</anno></c> is retrived + via the callback + <seealso marker="erts:alt_dist#hs_data_f_handshake_complete"><c>f_handshake_complete</c></seealso>. + More information can be found in the documentation of + <seealso marker="erts:alt_dist#distribution_module">ERTS + User's Guide ➜ How to implement an Alternative Carrier + for the Erlang Distribution ➜ Distribution Module</seealso>. + </p> + </desc> + </func> + + <func> <name name="element" arity="2"/> <fsummary>Return the Nth element of a tuple.</fsummary> <type_desc variable="N">1..tuple_size(<anno>Tuple</anno>)</type_desc> @@ -1943,23 +2104,26 @@ os_prompt%</pre> <item>The runtime system exits with integer value <c><anno>Status</anno></c> as status code to the calling environment (OS). + <note> + <p>On many platforms, the OS supports only status + codes 0-255. A too large status code is truncated by clearing + the high bits.</p> + </note> </item> <tag>string()</tag> <item>An Erlang crash dump is produced with <c><anno>Status</anno></c> as slogan. Then the runtime system exits with status code <c>1</c>. - Note that only code points in the range 0-255 may be used - and the string will be truncated if longer than 200 characters. + The string will be truncated if longer than 200 characters. + <note> + <p>Before ERTS 9.1 (OTP-20.1) only code points in the range 0-255 + was accepted in the string. Now any unicode string is valid.</p> + </note> </item> <tag><c>abort</c></tag> <item>The runtime system aborts producing a core dump, if that is enabled in the OS. </item> </taglist> - <note> - <p>On many platforms, the OS supports only status - codes 0-255. A too large status code is truncated by clearing - the high bits.</p> - </note> <p>For integer <c><anno>Status</anno></c>, the Erlang runtime system closes all ports and allows async threads to finish their operations before exiting. To exit without such flushing, use @@ -2125,6 +2289,15 @@ os_prompt%</pre> </func> <func> + <name name="iolist_to_iovec" arity="1"/> + <fsummary>Converts an iolist to a iovec.</fsummary> + <desc> + <p>Returns an iovec that is made from the integers and binaries in + <c><anno>IoListOrBinary</anno></c>.</p> + </desc> + </func> + + <func> <name name="is_alive" arity="0"/> <fsummary>Check whether the local node is alive.</fsummary> <desc> @@ -2458,6 +2631,15 @@ os_prompt%</pre> but only if there already exists such atom.</p> <p>Failure: <c>badarg</c> if there does not already exist an atom whose text representation is <c><anno>String</anno></c>.</p> + <note> + <p>Note that the compiler may optimize away atoms. For + example, the compiler will rewrite + <c>atom_to_list(some_atom)</c> to <c>"some_atom"</c>. If + that expression is the only mention of the atom + <c>some_atom</c> in the containing module, the atom will not + be created when the module is loaded, and a subsequent call + to <c>list_to_existing_atom("some_atom")</c> will fail.</p> + </note> </desc> </func> @@ -4284,7 +4466,6 @@ RealSystem = system + MissedSystem</code> <desc> <p><c><anno>Locking</anno></c> is one of the following:</p> <list type="bulleted"> - <item><c>false</c> (emulator without SMP support)</item> <item><c>port_level</c> (port-specific locking)</item> <item><c>driver_level</c> (driver-specific locking)</item> </list> @@ -4688,8 +4869,8 @@ RealSystem = system + MissedSystem</code> selected for execution. Notice however that this does <em>not</em> mean that no processes on priority <c>low</c> or <c>normal</c> can run when processes - are running on priority <c>high</c>. On the runtime - system with SMP support, more processes can be running + are running on priority <c>high</c>. When using multiple + schedulers, more processes can be running in parallel than processes on priority <c>high</c>. That is, a <c>low</c> and a <c>high</c> priority process can execute at the same time.</p> @@ -4704,10 +4885,8 @@ RealSystem = system + MissedSystem</code> execution.</p> <note> <p>Do not depend on the scheduling - to remain exactly as it is today. Scheduling, at least on - the runtime system with SMP support, is likely to be - changed in a future release to use available - processor cores better.</p> + to remain exactly as it is today. Scheduling is likely to be + changed in a future release to use available processor cores better.</p> </note> <p>There is <em>no</em> automatic mechanism for avoiding priority inversion, such as priority inheritance @@ -6219,8 +6398,7 @@ true</pre> <p><c>statistics(exact_reductions)</c> is a more expensive operation than <seealso marker="#statistics_reductions"> - statistics(reductions)</seealso>, - especially on an Erlang machine with SMP support.</p> + statistics(reductions)</seealso>.</p> </note> </desc> </func> @@ -6297,17 +6475,24 @@ lists:map( <p><c><anno>MSAcc_Thread_Type</anno></c>s:</p> <taglist> <tag><c>scheduler</c></tag> - <item>The main execution threads that do most of the work.</item> + <item>The main execution threads that do most of the work. See + <seealso marker="erts:erl#+S">erl +S</seealso> for more details.</item> <tag><c>dirty_cpu_scheduler</c></tag> - <item>The threads for long running cpu intensive work.</item> + <item>The threads for long running cpu intensive work. See + <seealso marker="erts:erl#+SDcpu">erl +SDcpu</seealso> for more details.</item> <tag><c>dirty_io_scheduler</c></tag> - <item>The threads for long running I/O work.</item> + <item>The threads for long running I/O work. See + <seealso marker="erts:erl#+SDio">erl +SDio</seealso> for more details.</item> <tag><c>async</c></tag> <item>Async threads are used by various linked-in drivers (mainly the - file drivers) do offload non-CPU intensive work.</item> + file drivers) do offload non-CPU intensive work. See + <seealso marker="erts:erl#+async_thread_pool_size">erl +A</seealso> for more details.</item> <tag><c>aux</c></tag> <item>Takes care of any work that is not specifically assigned to a scheduler.</item> + <tag><c>poll</c></tag> + <item>Does the IO polling for the emulator. See + <seealso marker="erts:erl#+IOt">erl +IOt</seealso> for more details.</item> </taglist> <p>The following <c><anno>MSAcc_Thread_State</anno></c>s are available. All states are exclusive, meaning that a thread cannot be in two @@ -6483,6 +6668,9 @@ lists:map( <p>This is the sum of the runtime for all threads in the Erlang runtime system and can therefore be greater than the wall clock time.</p> + <warning><p>This value might wrap due to limitations in the + underlying functionality provided by the operating system + that is used.</p></warning> <p>Example:</p> <pre> > <input>statistics(runtime).</input> @@ -6601,8 +6789,8 @@ ok than available logical processors, this value may be greater than <c>1.0</c>.</p> <p>As of ERTS version 9.0, the Erlang runtime system - with SMP support will as default have more schedulers - than logical processors. This due to the dirty schedulers.</p> + will as default have more schedulers than logical processors. + This due to the dirty schedulers.</p> <note> <p><c>scheduler_wall_time</c> is by default disabled. To enable it, use @@ -7861,15 +8049,6 @@ ok The return value will always be <c>false</c>, as the <c>elib_malloc</c> allocator has been removed.</p> </item> - <tag><marker id="system_info_eager_check_io"/> - <c>eager_check_io</c></tag> - <item> - <p>Returns the value of command-line flag - <seealso marker="erl#+secio"><c>+secio</c></seealso> in - <c>erl(1)</c>, which is either <c>true</c> or <c>false</c>. - For information about the different values, see the - documentation of the command-line flag.</p> - </item> <tag><c>ets_limit</c></tag> <item> <p>Returns the maximum number of ETS tables allowed. This @@ -7972,9 +8151,7 @@ ok <taglist> <tag><c>disabled</c></tag> <item> - <p>The emulator has only one scheduler thread. The - emulator does not have SMP support, or have been - started with only one scheduler thread.</p> + <p>The emulator has been started with only one scheduler thread.</p> </item> <tag><c>blocked</c></tag> <item> @@ -8337,8 +8514,7 @@ ok </item> <tag><c>smp_support</c></tag> <item> - <p>Returns <c>true</c> if the emulator has been compiled - with SMP support, otherwise <c>false</c> is returned.</p> + <p>Returns <c>true</c>.</p> </item> <tag><marker id="system_info_start_time"/><c>start_time</c></tag> <item> @@ -8361,8 +8537,7 @@ ok </item> <tag><c>threads</c></tag> <item> - <p>Returns <c>true</c> if the emulator has been compiled - with thread support, otherwise <c>false</c> is returned.</p> + <p>Returns <c>true</c>.</p> </item> <tag><c>thread_pool_size</c></tag> <item> @@ -10481,9 +10656,9 @@ true</pre> <c>receive after 1 -> ok end</c>, except that <c>yield()</c> is faster.</p> <warning> - <p>There is seldom or never any need to use this BIF, - especially in the SMP emulator, as other processes have a - chance to run in another scheduler thread anyway. + <p>There is seldom or never any need to use this BIF + as other processes have a chance to run in another scheduler + thread anyway. Using this BIF without a thorough grasp of how the scheduler works can cause performance degradation.</p> </warning> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index d3f725ef99..580780e73b 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -87,9 +87,9 @@ the number of system calls made.</item> </taglist> - <p><c>sys_alloc</c> and <c>literal_alloc</c> are always enabled and - cannot be disabled. <c>exec_alloc</c> is only available if it is needed - and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is + <p><c>sys_alloc</c>, <c>literal_alloc</c> and <c>temp_alloc</c> are always + enabled and cannot be disabled. <c>exec_alloc</c> is only available if it + is needed and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is available and an allocator that uses it is enabled. All other allocators can be <seealso marker="#M_e">enabled or disabled</seealso>. By default all allocators are enabled. diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 4d7e578738..9968b531f9 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,11 +31,363 @@ </header> <p>This document describes the changes made to the ERTS application.</p> -<section><title>Erts 9.0</title> +<section><title>Erts 9.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> <item> + <p>The new zlib module returned a data_error when + inflating concatenated streams, which was incompatible + with the old module's behavior of returning the + uncompressed data up to the end of the first stream.</p> + <p> + Own Id: OTP-14648</p> + </item> + <item> + <p>zlib:gunzip/1 will no longer stop at the end of the + first stream when decompressing concatenated gzip + files.</p> + <p> + Own Id: OTP-14649</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Changed <c>erlang:apply/2</c> to raise a <c>badarg</c> + exception if the second argument is not a proper list. + Previous behavior was a misleading <c>undef</c> + exception.</p> + <p> + Own Id: OTP-14490 Aux Id: ERL-432 </p> + </item> + <item> + <p>On macOS, <c>crypto</c> would crash if <c>observer</c> + had been started before <c>crypto</c>. On the beta for + macOS 10.13 (High Sierra), <c>crypto</c> would crash. + Both of those bugs have been fixed.</p> + <p> + Own Id: OTP-14499 Aux Id: ERL-251 ERL-439 </p> + </item> + <item> + <p> + Fixed bug in enif_whereis_pid/port that could cause heap + corruption in rare cases.</p> + <p> + Own Id: OTP-14523</p> + </item> + <item> + <p> + Fix so that trace messages generated when in a dirty nif + are flushed correctly when the dirty nif is done + executing.</p> + <p> + Own Id: OTP-14538</p> + </item> + <item> + <p> + Fix escape code handling when using ANSI color codes in + the shell.</p> + <p> + Own Id: OTP-14549 Aux Id: PR1536 </p> + </item> + <item> + <p> + Upgraded the ERTS internal PCRE library from version 8.40 + to version 8.41. See <url + href="http://pcre.org/original/changelog.txt">http://pcre.org/original/changelog.txt</url> + for information about changes made to PCRE. This library + implements major parts of the <seealso + marker="stdlib:re"><c>re</c></seealso> regular + expressions module.</p> + <p> + Own Id: OTP-14574</p> + </item> + <item> + <p> + Fixed a bug causing <c>statistics(runtime)</c> to produce + negative values and a bug in + <c>statistics(wall_clock)</c> causing it to produce + values one second too long.</p> + <p> + <c>statistics(runtime)</c> now also use + <c>getrusage()</c> as source when available preventing + the returned value from wrapping as frequent as before.</p> + <p> + Own Id: OTP-14597 Aux Id: ERL-465 </p> + </item> + <item> + <p> + Fixed small memory leak that could occur when sending to + a terminating port.</p> + <p> + Own Id: OTP-14609</p> + </item> + <item> + <p> + Fix bug causing VM crash when a module with + <c>-on_load</c> directive is loaded while + <c>erlang:trace(on_load, ...)</c> is enabled.</p> + <p> + Own Id: OTP-14612</p> + </item> + <item> + <p>A warning that the compiler may optimize away atoms + have been added to the documentation of + <c>list_to_existing_atom/1</c> and + <c>binary_to_existing_atom/2</c>.</p> + <p> + Own Id: OTP-14614 Aux Id: ERL-453 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>Lock counting can now be fully toggled at runtime in + the lock counting emulator (<c>-emu_type lcnt</c>). + Everything is enabled by default to match the old + behavior, but specific categories can be toggled at will + with minimal runtime overhead when disabled. Refer to the + documentation on <c>lcnt:rt_mask/1</c> for details.</p> + <p> + Own Id: OTP-13170</p> + </item> + <item> + <p>The <c>zlib</c> module has been refactored and all its + operations will now yield appropriately, allowing them to + be used freely in concurrent applications.</p> <p>The + following functions have been deprecated, but will not + produce compiler warnings until OTP 21: + <c>zlib:adler32</c>, <c>zlib:crc32</c>, + <c>zlib:inflateChunk</c>, <c>zlib:getBufSize</c>, + <c>zlib:setBufSize</c>.</p> <p>The behavior of throwing + an error when a dictionary is required for decompression + has also been deprecated. Refer to the documentation on + <c>inflateSetDictionary/2</c> for details.</p> + <p> + Own Id: OTP-14185</p> + </item> + <item> + <p><c>lcnt:collect</c> and <c>lcnt:clear</c> will no + longer block all other threads in the runtime system.</p> + <p> + Own Id: OTP-14412</p> + </item> + <item> + <p>Add <c>erlang:iolist_to_iovec/1</c>, which converts an + iolist() to an erlang:iovec(), which suitable for use + with <c>enif_inspect_iovec</c>.</p> + <p> + Own Id: OTP-14520</p> + </item> + <item> + <p>When provided with bad arguments, the <c>zlib</c> + module will now raise named exceptions instead of just + <c>badarg</c>. For example, <c>not_initialized</c> when + using <c>zlib:inflate/2</c> with an uninitialized + stream.</p> + <p> + Own Id: OTP-14527</p> + </item> + <item> + <p> + <c>erlang:halt/2</c> allows any Unicode string as slogan + for the crash dump.</p> + <p> + Own Id: OTP-14553</p> + </item> + <item> + <p>Add new nif API functions for managing an I/O Queue. + The added functions are:</p> <list type="bulleted"> + <item><seealso marker="erl_nif#enif_ioq_create"> + <c>enif_ioq_create()</c></seealso></item> <item><seealso + marker="erl_nif#enif_ioq_destroy"> + <c>enif_ioq_destroy()</c></seealso></item> <item><seealso + marker="erl_nif#enif_ioq_enq_binary"> + <c>enif_ioq_enq_binary()</c></seealso></item> + <item><seealso marker="erl_nif#enif_ioq_enqv"> + <c>enif_ioq_enqv()</c></seealso></item> <item><seealso + marker="erl_nif#enif_ioq_deq"> + <c>enif_ioq_deq()</c></seealso></item> <item><seealso + marker="erl_nif#enif_ioq_peek"> + <c>enif_ioq_peek()</c></seealso></item> <item><seealso + marker="erl_nif#enif_inspect_iovec"> + <c>enif_inspect_iovec()</c></seealso></item> + <item><seealso marker="erl_nif#enif_free_iovec"> + <c>enif_free_iovec()</c></seealso></item> </list> + <p> + Own Id: OTP-14598</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed bug in <c>binary_to_term</c> and + <c>binary_to_atom</c> that could cause VM crash. + Typically happens when the last character of an UTF8 + string is in the range 128 to 255, but truncated to only + one byte. Bug exists in <c>binary_to_term</c> since ERTS + version 5.10.2 (OTP_R16B01) and <c>binary_to_atom</c> + since ERTS version 9.0 (OTP-20.0).</p> + <p> + Own Id: OTP-14590 Aux Id: ERL-474 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A timer internal bit-field used for storing scheduler id + was too small. As a result, VM internal timer data + structures could become inconsistent when using 1024 + schedulers on the system. Note that systems with less + than 1024 schedulers are not effected by this bug.</p> + <p> + This bug was introduced in ERTS version 7.0 (OTP 18.0).</p> + <p> + Own Id: OTP-14548 Aux Id: OTP-11997, ERL-468 </p> + </item> + <item> + <p> + Automatic cleanup of a BIF timer, when the owner process + terminated, could race with the timeout of the timer. + This could cause the VM internal data structures to + become inconsistent which very likely caused a VM crash.</p> + <p> + This bug was introduced in ERTS version 9.0 (OTP 20.0).</p> + <p> + Own Id: OTP-14554 Aux Id: OTP-14356, ERL-468 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Binary append operations did not check for overflow, + resulting in nonsensical results when huge binaries were + appended.</p> + <p> + Own Id: OTP-14524</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Added missing release notes for OTP-14491 ("performance + bug in pre-allocators") which was included in erts-9.0.1 + (OTP-20.0.1).</p> + <p> + Own Id: OTP-14494</p> + </item> + <item> + <p>Fixed a bug that prevented TCP sockets from being + closed properly on send timeouts.</p> + <p> + Own Id: OTP-14509</p> + </item> + <item> + <p> + Fixed bug in operator <c>bxor</c> causing erroneuos + result when one operand is a big <em>negative</em> + integer with the lowest <c>N*W</c> bits as zero and the + other operand not larger than <c>N*W</c> bits. <c>N</c> + is an integer of 1 or larger and <c>W</c> is 32 or 64 + depending on word size.</p> + <p> + Own Id: OTP-14514</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in gen_tcp:send where it never returned when + repeatedly called on a remotely closed TCP socket.</p> + <p> + Own Id: OTP-13939 Aux Id: ERL-193 </p> + </item> + <item> + <p> + Fixed segfault that could happen during cleanup of + aborted erlang:port_command/3 calls. A port_command is + aborted if the port is closed at the same time as the + port_command was issued. This bug was introduced in + erts-8.0.</p> + <p> + Own Id: OTP-14481</p> + </item> + <item> + <p> + Fixed implementation of <c>statistics(wall_clock)</c> and + <c>statistics(runtime)</c> so that values do not + unnecessarily wrap due to the emulator. Note that the + values returned by <c>statistics(runtime)</c> may still + wrap due to limitations in the underlying functionality + provided by the operating system.</p> + <p> + Own Id: OTP-14484</p> + </item> + <item> + <p> + Fix performance bug in pre-allocators that could cause + them to permanently fall back on normal more expensive memory + allocation. Pre-allocators are used for quick allocation + of short lived meta data used by messages and other + scheduled tasks. Bug exists since OTP_R15B02. + [this release note was missing in erts-9.0.1]</p> + <p> + Own Id: OTP-14491</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.0</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> <p>Fix various bugs regarding loading, upgrade and purge of HiPE compiled code:</p> <list> <item>The native code memory for a purged module was never deallocated.</item> @@ -474,7 +826,7 @@ marker="erts:erl"><c>erl</c></seealso> command.</p> <p> See <url - href="http://pcre.org/original/changelog.txt"><c>http://pcre.org/original/changelog.txt</c></url> + href="http://pcre.org/original/changelog.txt">http://pcre.org/original/changelog.txt</url> for information about changes made to PCRE between the versions 8.33 and 8.40.</p> <p> @@ -631,6 +983,42 @@ </section> +<section><title>Erts 8.3.5.1</title> + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in gen_tcp:send where it never returned when + repeatedly called on a remotely closed TCP socket.</p> + <p> + Own Id: OTP-13939 Aux Id: ERL-193 </p> + </item> + <item> + <p> + Fixed segfault that could happen during cleanup of + aborted erlang:port_command/3 calls. A port_command is + aborted if the port is closed at the same time as the + port_command was issued. This bug was introduced in + erts-8.0.</p> + <p> + Own Id: OTP-14481</p> + </item> + <item> + <p> + Fixed implementation of <c>statistics(wall_clock)</c> and + <c>statistics(runtime)</c> so that values do not + unnecessarily wrap due to the emulator. Note that the + values returned by <c>statistics(runtime)</c> may still + wrap due to limitations in the underlying functionality + provided by the operating system.</p> + <p> + Own Id: OTP-14484</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.3.5</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -3376,8 +3764,7 @@ <p>The previously introduced "eager check I/O" feature is now enabled by default.</p> <p>Eager check I/O can be disabled using the <c>erl</c> - command line argument: <seealso - marker="erl#+secio"><c>+secio false</c></seealso></p> + command line argument: <c>+secio false</c></p> <p>Characteristics impact compared to previous default:</p> <list> <item>Lower latency and smoother management of externally triggered I/O operations.</item> @@ -4158,8 +4545,7 @@ prioritized to the same extent as when eager check I/O is disabled.</p> <p>Eager check I/O can be enabled using the <c>erl</c> - command line argument: <seealso - marker="erl#+secio"><c>+secio true</c></seealso></p> + command line argument: <c>+secio true</c></p> <p>Characteristics impact when enabled:</p> <list> <item>Lower latency and smoother management of externally triggered I/O operations.</item> <item>A slightly reduced diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml index 1d272c4c18..f5cc1b1e64 100644 --- a/erts/doc/src/zlib.xml +++ b/erts/doc/src/zlib.xml @@ -65,13 +65,17 @@ list_to_binary([Compressed|Last])</pre> <tag><c>badarg</c></tag> <item>Bad argument. </item> + <tag><c>not_initialized</c></tag> + <item>The stream hasn't been initialized, eg. if + <seealso marker="#inflateInit/1"><c>inflateInit/1</c></seealso> wasn't + called prior to a call to + <seealso marker="#inflate/2"><c>inflate/2</c></seealso>. + </item> <tag><c>data_error</c></tag> <item>The data contains errors. </item> <tag><c>stream_error</c></tag> <item>Inconsistent stream state.</item> - <tag><c>einval</c></tag> - <item>Bad value or wrong function called.</item> <tag><c>{need_dictionary,Adler32}</c></tag> <item>See <seealso marker="#inflate/2"><c>inflate/2</c></seealso>. </item> @@ -90,6 +94,9 @@ list_to_binary([Compressed|Last])</pre> <name name="zlevel"/> </datatype> <datatype> + <name name="zflush"/> + </datatype> + <datatype> <name name="zmemlevel"/> </datatype> <datatype> @@ -112,6 +119,11 @@ list_to_binary([Compressed|Last])</pre> <fsummary>Calculate the Adler checksum.</fsummary> <desc> <p>Calculates the Adler-32 checksum for <c><anno>Data</anno></c>.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#adler32/1"> + <c>erlang:adler32/1</c></seealso> instead.</p> + </warning> </desc> </func> @@ -127,6 +139,11 @@ list_to_binary([Compressed|Last])</pre> Crc = lists:foldl(fun(Data,Crc0) -> zlib:adler32(Z, Crc0, Data), end, zlib:adler32(Z,<< >>), Datas)</pre> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#adler32/2"> + <c>erlang:adler32/2</c></seealso> instead.</p> + </warning> </desc> </func> @@ -141,6 +158,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> <p>This function returns the <c><anno>Adler</anno></c> checksum of <c>[Data1,Data2]</c>, requiring only <c><anno>Adler1</anno></c>, <c><anno>Adler2</anno></c>, and <c><anno>Size2</anno></c>.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#adler32_combine/3"> + <c>erlang:adler32_combine/3</c></seealso> instead.</p> + </warning> </desc> </func> @@ -165,6 +187,12 @@ Crc = lists:foldl(fun(Data,Crc0) -> <fsummary>Get current CRC.</fsummary> <desc> <p>Gets the current calculated CRC checksum.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#crc32/1"> + <c>erlang:crc32/1</c></seealso> on the uncompressed data + instead.</p> + </warning> </desc> </func> @@ -173,6 +201,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> <fsummary>Calculate CRC.</fsummary> <desc> <p>Calculates the CRC checksum for <c><anno>Data</anno></c>.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#crc32/1"> + <c>erlang:crc32/1</c></seealso> instead.</p> + </warning> </desc> </func> @@ -188,6 +221,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> Crc = lists:foldl(fun(Data,Crc0) -> zlib:crc32(Z, Crc0, Data), end, zlib:crc32(Z,<< >>), Datas)</pre> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#crc32/2"> + <c>erlang:crc32/2</c></seealso> instead.</p> + </warning> </desc> </func> @@ -202,6 +240,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> <p>This function returns the <c><anno>CRC</anno></c> checksum of <c>[Data1,Data2]</c>, requiring only <c><anno>CRC1</anno></c>, <c><anno>CRC2</anno></c>, and <c><anno>Size2</anno></c>.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="erts:erlang#crc32_combine/3"> + <c>erlang:crc32_combine/3</c></seealso> instead.</p> + </warning> </desc> </func> @@ -407,8 +450,8 @@ list_to_binary([B1,B2])</pre> <seealso marker="#deflateInit/1"><c>deflateInit/1,2,6</c></seealso> or <seealso marker="#deflateReset/1"><c>deflateReset/1</c></seealso>, before any call of - <seealso marker="#deflate/3"><c>deflate/3</c></seealso>. - The compressor and decompressor must use the same dictionary (see + <seealso marker="#deflate/3"><c>deflate/3</c></seealso>.</p> + <p>The compressor and decompressor must use the same dictionary (see <seealso marker="#inflateSetDictionary/2"> <c>inflateSetDictionary/2</c></seealso>).</p> <p>The Adler checksum of the dictionary is returned.</p> @@ -420,6 +463,10 @@ list_to_binary([B1,B2])</pre> <fsummary>Get buffer size.</fsummary> <desc> <p>Gets the size of the intermediate buffer.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release.</p> + </warning> </desc> </func> @@ -443,14 +490,31 @@ list_to_binary([B1,B2])</pre> <name name="inflate" arity="2"/> <fsummary>Decompress data.</fsummary> <desc> - <p>Decompresses as much data as possible. - It can introduce some output latency (reading - input without producing any output).</p> - <p>If a preset dictionary is needed at this point (see - <seealso marker="#inflateSetDictionary/2"> - <c>inflateSetDictionary/2</c></seealso>), <c>inflate/2</c> throws a - <c>{need_dictionary,Adler}</c> exception, where <c>Adler</c> is - the Adler-32 checksum of the dictionary chosen by the compressor.</p> + <p>Equivalent to + <seealso marker="#inflate/3"><c>inflate(Z, Data, [])</c></seealso> + </p> + </desc> + </func> + + <func> + <name name="inflate" arity="3"/> + <fsummary>Decompress data.</fsummary> + <desc> + <p>Decompresses as much data as possible. It can introduce some output + latency (reading input without producing any output).</p> + <p>Currently the only available option is + <c>{exception_on_need_dict,boolean()}</c> which controls whether the + function should throw an exception when a preset dictionary is + required for decompression. When set to false, a + <c>need_dictionary</c> tuple will be returned instead. See + <seealso marker="#inflateSetDictionary/2"> + <c>inflateSetDictionary/2</c></seealso> for details.</p> + <warning> + <p>This option defaults to <c>true</c> for backwards compatibility + but we intend to remove the exception behavior in a future + release. New code that needs to handle dictionaries manually + should always specify <c>{exception_on_need_dict,false}</c>.</p> + </warning> </desc> </func> @@ -458,6 +522,11 @@ list_to_binary([B1,B2])</pre> <name name="inflateChunk" arity="1"/> <fsummary>Read next uncompressed chunk.</fsummary> <desc> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="#safeInflate/2"><c>safeInflate/2</c> + </seealso> instead.</p> + </warning> <p>Reads the next chunk of uncompressed data, initialized by <seealso marker="#inflateChunk/2"><c>inflateChunk/2</c></seealso>.</p> <p>This function is to be repeatedly called, while it returns @@ -469,23 +538,27 @@ list_to_binary([B1,B2])</pre> <name name="inflateChunk" arity="2"/> <fsummary>Decompress data with limited output size.</fsummary> <desc> + <warning> + <p>This function is deprecated and will be removed in a future + release. Use <seealso marker="#safeInflate/2"><c>safeInflate/2</c> + </seealso> instead.</p> + </warning> <p>Like <seealso marker="#inflate/2"><c>inflate/2</c></seealso>, - but decompresses no more data than will fit in the buffer configured - through <seealso marker="#setBufSize/2"><c>setBufSize/2</c></seealso>. - Is is useful when decompressing a stream with a high compression - ratio, such that a small amount of compressed input can expand up to - 1000 times.</p> + but decompresses no more data than will fit in the buffer configured + through <seealso marker="#setBufSize/2"><c>setBufSize/2</c> + </seealso>. Is is useful when decompressing a stream with a high + compression ratio, such that a small amount of compressed input can + expand up to 1000 times.</p> <p>This function returns <c>{more, Decompressed}</c>, when there is - more output available, and - <seealso marker="#inflateChunk/1"><c>inflateChunk/1</c></seealso> - is to be used to read it.</p> - <p>This function can introduce some output latency (reading - input without producing any output).</p> - <p>If a preset dictionary is needed at this point (see - <seealso marker="#inflateSetDictionary/2"> - <c>inflateSetDictionary/2</c></seealso>), this function throws a - <c>{need_dictionary,Adler}</c> exception, where <c>Adler</c> is - the Adler-32 checksum of the dictionary chosen by the compressor.</p> + more output available, and + <seealso marker="#inflateChunk/1"><c>inflateChunk/1</c></seealso> + is to be used to read it.</p> + <p>This function can introduce some output latency (reading input + without producing any output).</p> + <p>An exception will be thrown if a preset dictionary is required for + further decompression. See + <seealso marker="#inflateSetDictionary/2"> + <c>inflateSetDictionary/2</c></seealso> for details.</p> <p>Example:</p> <pre> walk(Compressed, Handler) -> @@ -517,6 +590,18 @@ loop(Z, Handler, Uncompressed) -> </func> <func> + <name name="inflateGetDictionary" arity="1"/> + <fsummary>Return the decompression dictionary.</fsummary> + <desc> + <p>Returns the decompression dictionary currently in use + by the stream. This function must be called between + <seealso marker="#inflateInit/1"><c>inflateInit/1,2</c></seealso> + and <seealso marker="#inflateEnd/1"><c>inflateEnd</c></seealso>.</p> + <p>Only supported if ERTS was compiled with zlib >= 1.2.8.</p> + </desc> + </func> + + <func> <name name="inflateInit" arity="1"/> <fsummary>Initialize a session for decompression.</fsummary> <desc> @@ -562,45 +647,83 @@ loop(Z, Handler, Uncompressed) -> <fsummary>Initialize the decompression dictionary.</fsummary> <desc> <p>Initializes the decompression dictionary from the specified - uncompressed byte sequence. This function must be called - immediately after a call of - <seealso marker="#inflate/2"><c>inflate/2</c></seealso> - if this call threw a <c>{need_dictionary,Adler}</c> exception. - The dictionary chosen by the compressor can be determined from the - Adler value thrown by the call to <c>inflate/2</c>. - The compressor and decompressor must use the same dictionary (see - <seealso marker="#deflateSetDictionary/2"> - <c>deflateSetDictionary/2</c></seealso>).</p> + uncompressed byte sequence. This function must be called as a + response to an inflate operation (eg. + <seealso marker="#safeInflate/2"><c>safeInflate/2</c></seealso>) + returning <c>{need_dictionary,Adler,Output}</c> or in the case of + deprecated functions, throwing an + <c>{'EXIT',{{need_dictionary,Adler},_StackTrace}}</c> exception.</p> + <p>The dictionary chosen by the compressor can be determined from the + Adler value returned or thrown by the call to the inflate function. + The compressor and decompressor must use the same dictionary (See + <seealso marker="#deflateSetDictionary/2"> + <c>deflateSetDictionary/2</c></seealso>).</p> + <p>After setting the dictionary the inflate operation should be + retried without new input.</p> <p>Example:</p> <pre> -unpack(Z, Compressed, Dict) -> +deprecated_unpack(Z, Compressed, Dict) -> case catch zlib:inflate(Z, Compressed) of - {'EXIT',{{need_dictionary,DictID},_}} -> - zlib:inflateSetDictionary(Z, Dict), + {'EXIT',{{need_dictionary,_DictID},_}} -> + ok = zlib:inflateSetDictionary(Z, Dict), Uncompressed = zlib:inflate(Z, []); Uncompressed -> Uncompressed - end.</pre> + end. + +new_unpack(Z, Compressed, Dict) -> + case zlib:inflate(Z, Compressed, [{exception_on_need_dict, false}]) of + {need_dictionary, _DictId, Output} -> + ok = zlib:inflateSetDictionary(Z, Dict), + [Output | zlib:inflate(Z, [])]; + Uncompressed -> + Uncompressed + end.</pre> </desc> </func> <func> - <name name="inflateGetDictionary" arity="1"/> - <fsummary>Return the decompression dictionary.</fsummary> + <name name="open" arity="0"/> + <fsummary>Open a stream and return a stream reference.</fsummary> <desc> - <p>Returns the decompression dictionary currently in use - by the stream. This function must be called between - <seealso marker="#inflateInit/1"><c>inflateInit/1,2</c></seealso> - and <seealso marker="#inflateEnd/1"><c>inflateEnd</c></seealso>.</p> - <p>Only supported if ERTS was compiled with zlib >= 1.2.8.</p> + <p>Opens a zlib stream.</p> </desc> </func> <func> - <name name="open" arity="0"/> - <fsummary>Open a stream and return a stream reference.</fsummary> + <name name="safeInflate" arity="2"/> + <fsummary>Decompress data with limited output size.</fsummary> <desc> - <p>Opens a zlib stream.</p> + <p>Like <seealso marker="#inflate/2"><c>inflate/2</c></seealso>, + but returns once it has expanded beyond a small + implementation-defined threshold. It's useful when decompressing + untrusted input which could have been maliciously crafted to expand + until the system runs out of memory.</p> + <p>This function returns <c>{continue | finished, Output}</c>, where + <anno>Output</anno> is the data that was decompressed in this call. + New input can be queued up on each call if desired, and the function + will return <c>{finished, Output}</c> once all queued data has been + decompressed.</p> + <p>This function can introduce some output latency (reading + input without producing any output).</p> + <p>If a preset dictionary is required for further decompression, this + function returns a <c>need_dictionary</c> tuple. See + <seealso marker="#inflateSetDictionary/2"> + <c>inflateSetDictionary/2</c></seealso>) for details.</p> + <p>Example:</p> + <pre> +walk(Compressed, Handler) -> + Z = zlib:open(), + zlib:inflateInit(Z), + loop(Z, Handler, zlib:safeInflate(Z, Compressed)), + zlib:inflateEnd(Z), + zlib:close(Z). + +loop(Z, Handler, {continue, Output}) -> + Handler(Output), + loop(Z, Handler, zlib:safeInflate(Z, [])); +loop(Z, Handler, {finished, Output}) -> + Handler(Output).</pre> </desc> </func> @@ -609,6 +732,10 @@ unpack(Z, Compressed, Dict) -> <fsummary>Set buffer size.</fsummary> <desc> <p>Sets the intermediate buffer size.</p> + <warning> + <p>This function is deprecated and will be removed in a future + release.</p> + </warning> </desc> </func> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 61c1e14741..915cad4e18 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -28,10 +28,22 @@ HIPE_ENABLED=@HIPE_ENABLED@ DTRACE_ENABLED=@DTRACE_ENABLED@ DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@ USE_VM_PROBES=@USE_VM_PROBES@ +FPE=@FPE@ LIBS = @LIBS@ Z_LIB=@Z_LIB@ NO_INLINE_FUNCTIONS=false -OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab +OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab \ +beam/ops.tab \ +beam/macros.tab \ +beam/instrs.tab \ +beam/arith_instrs.tab \ +beam/bif_instrs.tab \ +beam/bs_instrs.tab \ +beam/float_instrs.tab \ +beam/map_instrs.tab \ +beam/msg_instrs.tab \ +beam/select_instrs.tab \ +beam/trace_instrs.tab DEBUG_CFLAGS = @DEBUG_CFLAGS@ CONFIGURE_CFLAGS = @CFLAGS@ @@ -51,7 +63,28 @@ ARFLAGS=rc OMIT_OMIT_FP=no TYPE_LIBS= -DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ +PROFILE_COMPILER=@PROFILE_COMPILER@ +PROFILE_MARKER= +ifeq ($(PROFILE),generate) +PROFILE_MARKER=_pg +else +ifeq ($(PROFILE),use) +PROFILE_MARKER=_pu +endif +endif + +ifeq ($(PROFILE_COMPILER), gcc) +PROFILE_CORRECTION=@PROFILE_CORRECTION@ +PROFILE_GENERATE=-fprofile-generate +PROFILE_USE=-fprofile-use $(PROFILE_CORRECTION) +PROFILE_USE_DEPS=$(OBJDIR)/%_pu.gcda +endif +ifeq ($(PROFILE_COMPILER), clang) +PROFILE_GENERATE=-fprofile-instr-generate +PROFILE_USE=-fprofile-instr-use=$(OBJDIR)/default.profdata +PROFILE_USE_DEPS=$(OBJDIR)/default.profdata +endif + DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@ ifeq ($(TYPE),debug) @@ -116,6 +149,7 @@ ifeq ($(TYPE),lcnt) PURIFY = TYPEMARKER = .lcnt TYPE_FLAGS = @CFLAGS@ -DERTS_ENABLE_LOCK_COUNT +ENABLE_ALLOC_TYPE_VARS += lcnt else ifeq ($(TYPE),frmptr) @@ -177,31 +211,10 @@ endif # NOTE: When adding a new type update ERL_BUILD_TYPE_MARKER in sys/unix/sys.c # -FLAVOR=$(DEFAULT_FLAVOR) - -ifeq ($(FLAVOR),plain) - -DS_SUPPORT=no -DS_TEST=no - -FLAVOR_MARKER= -FLAVOR_FLAGS= -ENABLE_ALLOC_TYPE_VARS += nofrag -M4FLAGS += - -else # FLAVOR - -# If flavor isn't one of the above, it *is* smp flavor... override FLAVOR=smp FLAVOR_MARKER=.smp -FLAVOR_FLAGS=-DERTS_SMP -ENABLE_ALLOC_TYPE_VARS += smp nofrag -M4FLAGS += -DERTS_SMP=1 -ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes) -THR_DEFS += -DERTS_DIRTY_SCHEDULERS -DS_SUPPORT=yes - +ENABLE_ALLOC_TYPE_VARS += nofrag ifeq ($(DIRTY_SCHEDULER_TEST),yes) DS_TEST=yes THR_DEFS += -DERTS_DIRTY_SCHEDULERS_TEST @@ -209,13 +222,6 @@ else # DIRTY_SCHEDULER_TEST DS_TEST=no endif # DIRTY_SCHEDULER_TEST -else # DIRTY_SCHEDULER_SUPPORT -DS_SUPPORT=no -DS_TEST=no -endif # DIRTY_SCHEDULER_SUPPORT - -endif # FLAVOR - TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no) @@ -245,7 +251,7 @@ endif HIPEBEAMLDFLAGS=$($(ARCH)BEAMLDFLAGS) endif -ERTS_ENABLE_KERNEL_POLL=@ERTS_ENABLE_KERNEL_POLL@ +ERTS_BUILD_FALLBACK_POLL=@ERTS_BUILD_FALLBACK_POLL@ # # @@ -424,9 +430,20 @@ ifeq ($(TARGET), win32) EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll else EMULATOR_EXECUTABLE = beam$(TF_MARKER) +PROFILE_EXECUTABLE = beam.prof$(TF_MARKER) endif CS_EXECUTABLE = erl_child_setup$(TYPEMARKER) +ifeq ($(PROFILE), generate) +EMULATOR_EXECUTABLE = $(PROFILE_EXECUTABLE) +ifeq ($(PROFILE_COMPILER), gcc) +PROFILE_LDFLAGS = -fprofile-generate +endif +ifeq ($(PROFILE_COMPILER), clang) +PROFILE_LDFLAGS = -fprofile-instr-generate +endif +endif + # ---------------------------------------------------------------------- ifeq ($(ERLANG_OSTYPE), unix) @@ -547,10 +564,11 @@ DTRACE_HEADERS = endif ifdef HIPE_ENABLED -OPCODE_TABLES += hipe/hipe_ops.tab +OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab endif $(TTF_DIR)/beam_cold.h \ +$(TTF_DIR)/beam_warm.h \ $(TTF_DIR)/beam_hot.h \ $(TTF_DIR)/beam_opcodes.c \ $(TTF_DIR)/beam_opcodes.h \ @@ -562,6 +580,7 @@ $(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops -wordsize @EXTERNAL_WORD_SIZE@ \ -outdir $(TTF_DIR) \ -DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \ + -DNO_FPE_SIGNALS=$(if $filter(unreliable,$(FPE)),1,0) \ -emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED GENERATE += $(TTF_DIR)/OPCODES-GENERATED @@ -598,7 +617,7 @@ $(HIPE_NBIF_FILES) \ : $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/TABLES-GENERATED: $(ATOMS) $(DIRTY_BIFS) $(BIFS) utils/make_tables $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TTF_DIR) -include $(TTF_DIR)\ - -ds $(DS_SUPPORT) -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED + -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED GENERATE += $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types @@ -692,16 +711,33 @@ $(OBJDIR)/beams.$(RES_EXT): $(TARGET)/beams.rc endif -ifneq ($(filter tile-%,$(TARGET)),) -$(OBJDIR)/beam_emu.o: beam/beam_emu.c - $(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) \ - $(INCLUDES) -c $< -o $@ -else # Usually the same as the default rule, but certain platforms (e.g. win32) mix # different compilers $(OBJDIR)/beam_emu.o: beam/beam_emu.c $(V_EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ -endif + +$(OBJDIR)/%_pg.o: beam/%.c + $(V_CC) $(PROFILE_GENERATE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ +$(OBJDIR)/%_pu.o: beam/%.c $(PROFILE_USE_DEPS) + $(V_CC) $(PROFILE_USE) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ + +$(OBJDIR)/PROFILE: $(BINDIR)/$(PROFILE_EXECUTABLE) + $(V_at)echo " PROFILE ${PROFILE_EXECUTABLE}" + $(V_at)rm -f $(OBJDIR)/erl*.profraw + $(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erlc-%m.profraw" \ + ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERLC) -DPGO \ + -o $(OBJDIR) test/estone_SUITE.erl > $(OBJDIR)/PROFILE_LOG + $(V_at)set -e; LLVM_PROFILE_FILE="$(OBJDIR)/erl-%m.profraw" \ + ERL_FLAGS="-emu_type prof${TYPEMARKER} +S 1" $(ERL) -pa $(OBJDIR) \ + -noshell -s estone_SUITE pgo -s init stop >> $(OBJDIR)/PROFILE_LOG + $(V_at)touch $@ + +$(OBJDIR)/%_pu.gcda: $(OBJDIR)/PROFILE + $(V_at)mv $(OBJDIR)/$*_pg.gcda $@ + $(V_at)touch $@ + +$(OBJDIR)/default.profdata: $(OBJDIR)/PROFILE + $(V_LLVM_PROFDATA) merge -output $@ $(OBJDIR)/*.profraw $(OBJDIR)/%.o: beam/%.c $(V_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ @@ -763,15 +799,23 @@ $(ERL_TOP)/lib/%.beam: INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD_OBJ) +# -fprofile-correction is needed in order to use PGO on erl_process +# as multiple threads execute in that file. +ifeq ($(PROFILE_CORRECTION),) +PROFILE_OBJS = $(OBJDIR)/beam_emu.o +RUN_OBJS = $(OBJDIR)/erl_process.o +else +PROFILE_OBJS = $(OBJDIR)/beam_emu.o $(OBJDIR)/erl_process.o +endif + EMU_OBJS = \ - $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ + $(OBJDIR)/beam_opcodes.o \ $(OBJDIR)/beam_load.o $(OBJDIR)/beam_bif_load.o \ $(OBJDIR)/beam_debug.o $(OBJDIR)/beam_bp.o \ - $(OBJDIR)/beam_catches.o \ - $(OBJDIR)/code_ix.o \ + $(OBJDIR)/beam_catches.o $(OBJDIR)/code_ix.o \ $(OBJDIR)/beam_ranges.o -RUN_OBJS = \ +RUN_OBJS += \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ $(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \ @@ -787,7 +831,7 @@ RUN_OBJS = \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ $(OBJDIR)/erl_debug.o $(OBJDIR)/erl_md5.o \ - $(OBJDIR)/erl_message.o $(OBJDIR)/erl_process.o \ + $(OBJDIR)/erl_message.o \ $(OBJDIR)/erl_process_dict.o $(OBJDIR)/erl_process_lock.o \ $(OBJDIR)/erl_port_task.o $(OBJDIR)/erl_arith.o \ $(OBJDIR)/time.o $(OBJDIR)/erl_time_sup.o \ @@ -814,17 +858,19 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_binary.o $(OBJDIR)/erl_ao_firstfit_alloc.o \ $(OBJDIR)/erl_thr_queue.o $(OBJDIR)/erl_sched_spec_pre_alloc.o \ $(OBJDIR)/erl_ptab.o $(OBJDIR)/erl_map.o \ - $(OBJDIR)/erl_msacc.o + $(OBJDIR)/erl_msacc.o $(OBJDIR)/erl_lock_flags.o \ + $(OBJDIR)/erl_io_queue.o LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o -NIF_OBJS = $(OBJDIR)/erl_tracer_nif.o +NIF_OBJS = \ + $(OBJDIR)/erl_tracer_nif.o \ + $(OBJDIR)/zlib_nif.o ifeq ($(TARGET),win32) DRV_OBJS = \ $(OBJDIR)/registry_drv.o \ $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/zlib_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o OS_OBJS = \ @@ -854,7 +900,6 @@ OS_OBJS = \ DRV_OBJS = \ $(OBJDIR)/efile_drv.o \ $(OBJDIR)/inet_drv.o \ - $(OBJDIR)/zlib_drv.o \ $(OBJDIR)/ram_file_drv.o \ $(OBJDIR)/ttsl_drv.o endif @@ -883,17 +928,15 @@ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS): echo "=== Leaving lib after making static libs" endif -ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes) +ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes) OS_OBJS += $(OBJDIR)/erl_poll.kp.o \ - $(OBJDIR)/erl_check_io.kp.o \ - $(OBJDIR)/erl_poll.nkp.o \ - $(OBJDIR)/erl_check_io.nkp.o + $(OBJDIR)/erl_poll.nkp.o else -OS_OBJS += $(OBJDIR)/erl_poll.o \ - $(OBJDIR)/erl_check_io.o +OS_OBJS += $(OBJDIR)/erl_poll.o endif -OS_OBJS += $(OBJDIR)/erl_mseg.o \ +OS_OBJS += $(OBJDIR)/erl_check_io.o \ + $(OBJDIR)/erl_mseg.o \ $(OBJDIR)/erl_mmap.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ $(OBJDIR)/erl_mtrace_sys_wrap.o \ @@ -928,21 +971,23 @@ ifdef HIPE_ENABLED EXTRA_BASE_OBJS += $(HIPE_OBJS) endif -BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) $(LTTNG_OBJS) +BASE_OBJS = $(EMU_OBJS) $(RUN_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) \ + $(LTTNG_OBJS) $(DRV_OBJS) $(NIF_OBJS) -before_DTrace_OBJS = $(BASE_OBJS) $(DRV_OBJS) $(NIF_OBJS) +PROF_OBJS = $(patsubst %.o,%$(PROFILE_MARKER).o,$(PROFILE_OBJS)) $(BASE_OBJS) + +OBJS = $(PROF_OBJS) -DTRACE_OBJS = ifdef DTRACE_ENABLED_2STEP -DTRACE_OBJS = $(OBJDIR)/erlang_dtrace.o -$(OBJDIR)/erlang_dtrace.o: $(before_DTrace_OBJS) $(TARGET)/erlang_dtrace.h +# The $(PROFILE_MARKER) is placed in the object file name in order to +# make sure we re-compile with the new object files for the profiled emulator +OBJS += $(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o +$(OBJDIR)/erlang$(PROFILE_MARKER)_dtrace.o: $(PROF_OBJS) $(TARGET)/erlang_dtrace.h dtrace -G -C -Ibeam \ -s beam/erlang_dtrace.d \ - -o $@ $(before_DTrace_OBJS) + -o $@ $(PROF_OBJS) endif -OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS) - $(INIT_OBJS): $(TTF_DIR)/GENERATED $(OBJS): $(TTF_DIR)/GENERATED @@ -1034,8 +1079,8 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) else $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) - $(ld_verbose)$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \ - $(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \ + $(ld_verbose)$(PURIFY) $(LD) -o $@ \ + $(HIPEBEAMLDFLAGS) $(PROFILE_LDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) \ $(STATIC_NIF_LIBS) $(STATIC_DRIVER_LIBS) $(LIBS) endif @@ -1065,7 +1110,7 @@ else SED_PREFIX= endif -ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes) +ifeq ($(ERTS_BUILD_FALLBACK_POLL),yes) SED_SUFFIX=;$(SED_REPL_POLL);$(SED_REPL_CHK_IO) else SED_SUFFIX= diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab new file mode 100644 index 0000000000..b828e86788 --- /dev/null +++ b/erts/emulator/beam/arith_instrs.tab @@ -0,0 +1,396 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) { + Eterm result; + Uint live = $Live; + HEAVY_SWAPOUT; + reg[live] = $Op1; + reg[live+1] = $Op2; + result = erts_gc_$Name (c_p, reg, live); + HEAVY_SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + $BIF_ERROR_ARITY_2($Fail, $BIF, reg[live], reg[live+1]); +} + + +i_plus := plus.fetch.execute; + +plus.head() { + Eterm PlusOp1, PlusOp2; +} + +plus.fetch(Op1, Op2) { + PlusOp1 = $Op1; + PlusOp2 = $Op2; +} + +plus.execute(Fail, Live, Dst) { + if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) { + Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); + if (ERTS_LIKELY(IS_SSMALL(i))) { + $Dst = make_small(i); + $NEXT0(); + } + } + $OUTLINED_ARITH_2($Fail, $Live, mixed_plus, BIF_splus_2, PlusOp1, PlusOp2, $Dst); +} + +i_minus := minus.fetch.execute; + +minus.head() { + Eterm MinusOp1, MinusOp2; +} + +minus.fetch(Op1, Op2) { + MinusOp1 = $Op1; + MinusOp2 = $Op2; +} + +minus.execute(Fail, Live, Dst) { + if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) { + Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); + if (ERTS_LIKELY(IS_SSMALL(i))) { + $Dst = make_small(i); + $NEXT0(); + } + } + $OUTLINED_ARITH_2($Fail, $Live, mixed_minus, BIF_sminus_2, MinusOp1, MinusOp2, $Dst); +} + +i_increment := increment.fetch.execute; + +increment.head() { + Eterm increment_reg_val; +} + +increment.fetch(Src) { + increment_reg_val = $Src; +} + +increment.execute(IncrementVal, Live, Dst) { + Eterm increment_val = $IncrementVal; + Uint live; + Eterm result; + + if (ERTS_LIKELY(is_small(increment_reg_val))) { + Sint i = signed_val(increment_reg_val) + increment_val; + if (ERTS_LIKELY(IS_SSMALL(i))) { + $Dst = make_small(i); + $NEXT0(); + } + } + live = $Live; + HEAVY_SWAPOUT; + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + HEAVY_SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; +} + +i_times(Fail, Live, Op1, Op2, Dst) { + Eterm op1 = $Op1; + Eterm op2 = $Op2; + $OUTLINED_ARITH_2($Fail, $Live, mixed_times, BIF_stimes_2, op1, op2, $Dst); +} + +i_m_div(Fail, Live, Op1, Op2, Dst) { + Eterm op1 = $Op1; + Eterm op2 = $Op2; + $OUTLINED_ARITH_2($Fail, $Live, mixed_div, BIF_div_2, op1, op2, $Dst); +} + +i_int_div(Fail, Live, Op1, Op2, Dst) { + Eterm op1 = $Op1; + Eterm op2 = $Op2; + if (ERTS_UNLIKELY(op2 == SMALL_ZERO)) { + c_p->freason = BADARITH; + $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2); + } else if (ERTS_LIKELY(is_both_small(op1, op2))) { + Sint ires = signed_val(op1) / signed_val(op2); + if (ERTS_LIKELY(IS_SSMALL(ires))) { + $Dst = make_small(ires); + $NEXT0(); + } + } + $OUTLINED_ARITH_2($Fail, $Live, int_div, BIF_intdiv_2, op1, op2, $Dst); +} + +i_rem := rem.fetch.execute; + +rem.head() { + Eterm RemOp1, RemOp2; +} + +rem.fetch(Src1, Src2) { + RemOp1 = $Src1; + RemOp2 = $Src2; +} + +rem.execute(Fail, Live, Dst) { + if (ERTS_UNLIKELY(RemOp2 == SMALL_ZERO)) { + c_p->freason = BADARITH; + $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2); + } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) { + $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2)); + $NEXT0(); + } else { + $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst); + } +} + +i_band := band.fetch.execute; + +band.head() { + Eterm BandOp1, BandOp2; +} + +band.fetch(Src1, Src2) { + BandOp1 = $Src1; + BandOp2 = $Src2; +} + +band.execute(Fail, Live, Dst) { + if (ERTS_LIKELY(is_both_small(BandOp1, BandOp2))) { + /* + * No need to untag -- TAG & TAG == TAG. + */ + $Dst = BandOp1 & BandOp2; + $NEXT0(); + } + $OUTLINED_ARITH_2($Fail, $Live, band, BIF_band_2, BandOp1, BandOp2, $Dst); +} + +i_bor(Fail, Live, Src1, Src2, Dst) { + if (ERTS_LIKELY(is_both_small($Src1, $Src2))) { + /* + * No need to untag -- TAG | TAG == TAG. + */ + $Dst = $Src1 | $Src2; + $NEXT0(); + } + $OUTLINED_ARITH_2($Fail, $Live, bor, BIF_bor_2, $Src1, $Src2, $Dst); +} + +i_bxor(Fail, Live, Src1, Src2, Dst) { + if (ERTS_LIKELY(is_both_small($Src1, $Src2))) { + /* + * TAG ^ TAG == 0. + * + * Therefore, we perform the XOR operation on the tagged values, + * and OR in the tag bits. + */ + $Dst = ($Src1 ^ $Src2) | make_small(0); + $NEXT0(); + } + $OUTLINED_ARITH_2($Fail, $Live, bxor, BIF_bxor_2, $Src1, $Src2, $Dst); +} + +i_bsl := shift.setup_bsl.execute; +i_bsr := shift.setup_bsr.execute; + +shift.head() { + Eterm Op1, Op2; + Sint shift_left_count; +} + +shift.setup_bsr(Src1, Src2) { + Op1 = $Src1; + Op2 = $Src2; + shift_left_count = 0; + if (ERTS_LIKELY(is_small(Op2))) { + shift_left_count = -signed_val(Op2); + } else if (is_big(Op2)) { + /* + * N bsr NegativeBigNum == N bsl MAX_SMALL + * N bsr PositiveBigNum == N bsl MIN_SMALL + */ + shift_left_count = make_small(bignum_header_is_neg(*big_val(Op2)) ? + MAX_SMALL : MIN_SMALL); + } +} + +shift.setup_bsl(Src1, Src2) { + Op1 = $Src1; + Op2 = $Src2; + shift_left_count = 0; + if (ERTS_LIKELY(is_small(Op2))) { + shift_left_count = signed_val(Op2); + } else if (is_big(Op2)) { + if (bignum_header_is_neg(*big_val(Op2))) { + /* + * N bsl NegativeBigNum is either 0 or -1, depending on + * the sign of N. Since we don't believe this case + * is common, do the calculation with the minimum + * amount of code. + */ + shift_left_count = MIN_SMALL; + } else if (is_integer(Op1)) { + /* + * N bsl PositiveBigNum is too large to represent. + */ + shift_left_count = MAX_SMALL; + } + } +} + +shift.execute(Fail, Live, Dst) { + Uint big_words_needed; + + if (ERTS_LIKELY(is_small(Op1))) { + Sint int_res = signed_val(Op1); + if (ERTS_UNLIKELY(shift_left_count == 0 || int_res == 0)) { + if (ERTS_UNLIKELY(is_not_integer(Op2))) { + goto shift_error; + } + if (int_res == 0) { + $Dst = Op1; + $NEXT0(); + } + } else if (shift_left_count < 0) { /* Right shift */ + Eterm bsr_res; + shift_left_count = -shift_left_count; + if (shift_left_count >= SMALL_BITS-1) { + bsr_res = (int_res < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; + } else { + bsr_res = make_small(int_res >> shift_left_count); + } + $Dst = bsr_res; + $NEXT0(); + } else if (shift_left_count < SMALL_BITS-1) { /* Left shift */ + if ((int_res > 0 && + ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & int_res) == 0) || + ((~(Uint)0 << ((SMALL_BITS-1)-shift_left_count)) & ~int_res) == 0) { + $Dst = make_small(int_res << shift_left_count); + $NEXT0(); + } + } + big_words_needed = 1; /* big_size(small_to_big(Op1)) */ + goto big_shift; + } else if (is_big(Op1)) { + if (shift_left_count == 0) { + if (is_not_integer(Op2)) { + goto shift_error; + } + $Dst = Op1; + $NEXT0(); + } + big_words_needed = big_size(Op1); + + big_shift: + if (shift_left_count > 0) { /* Left shift. */ + big_words_needed += (shift_left_count / D_EXP); + } else { /* Right shift. */ + if (big_words_needed <= (-shift_left_count / D_EXP)) { + big_words_needed = 3; /* ??? */ + } else { + big_words_needed -= (-shift_left_count / D_EXP); + } + } + { + Eterm tmp_big[2]; + Sint big_need_size = BIG_NEED_SIZE(big_words_needed+1); + + /* + * Slightly conservative check the size to avoid + * allocating huge amounts of memory for bignums that + * clearly would overflow the arity in the header + * word. + */ + if (big_need_size-8 > BIG_ARITY_MAX) { + $SYSTEM_LIMIT($Fail); + } + $GC_TEST_PRESERVE(big_need_size+1, $Live, Op1); + if (is_small(Op1)) { + Op1 = small_to_big(signed_val(Op1), tmp_big); + } + Op1 = big_lshift(Op1, shift_left_count, HTOP); + if (is_big(Op1)) { + HTOP += bignum_header_arity(*HTOP) + 1; + } + HEAP_SPACE_VERIFIED(0); + if (ERTS_UNLIKELY(is_nil(Op1))) { + /* + * This result must have been only slighty larger + * than allowed since it wasn't caught by the + * previous test. + */ + $SYSTEM_LIMIT($Fail); + } + ERTS_HOLE_CHECK(c_p); + $REFRESH_GEN_DEST(); + $Dst = Op1; + $NEXT0(); + } + } + + /* + * One or more non-integer arguments. + */ + shift_error: + c_p->freason = BADARITH; + if ($Fail) { + $FAIL($Fail); + } else { + reg[0] = Op1; + reg[1] = Op2; + SWAPOUT; + if (IsOpCode(I[0], i_bsl_ssjtd)) { + I = handle_error(c_p, I, reg, &bif_export[BIF_bsl_2]->info.mfa); + } else { + ASSERT(IsOpCode(I[0], i_bsr_ssjtd)); + I = handle_error(c_p, I, reg, &bif_export[BIF_bsr_2]->info.mfa); + } + goto post_error_handling; + } +} + +i_int_bnot(Fail, Src, Live, Dst) { + Eterm bnot_val = $Src; + if (ERTS_LIKELY(is_small(bnot_val))) { + bnot_val = make_small(~signed_val(bnot_val)); + } else { + Uint live = $Live; + HEAVY_SWAPOUT; + reg[live] = bnot_val; + bnot_val = erts_gc_bnot(c_p, reg, live); + HEAVY_SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (ERTS_UNLIKELY(is_nil(bnot_val))) { + $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]); + } + $REFRESH_GEN_DEST(); + } + $Dst = bnot_val; +} diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 2055c29190..bbe1cb3e11 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -34,20 +34,18 @@ IndexTable erts_atom_table; /* The index table */ -#include "erl_smp.h" +static erts_rwmtx_t atom_table_lock; -static erts_smp_rwmtx_t atom_table_lock; - -#define atom_read_lock() erts_smp_rwmtx_rlock(&atom_table_lock) -#define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) -#define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) -#define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) +#define atom_read_lock() erts_rwmtx_rlock(&atom_table_lock) +#define atom_read_unlock() erts_rwmtx_runlock(&atom_table_lock) +#define atom_write_lock() erts_rwmtx_rwlock(&atom_table_lock) +#define atom_write_unlock() erts_rwmtx_rwunlock(&atom_table_lock) #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif #ifdef ERTS_ATOM_PUT_OPS_STAT -static erts_smp_atomic_t atom_put_ops; +static erts_atomic_t atom_put_ops; #endif /* Functions for allocating space for the ext of atoms. We do not @@ -76,7 +74,7 @@ void atom_info(fmtfn_t to, void *to_arg) index_info(to, to_arg, &erts_atom_table); #ifdef ERTS_ATOM_PUT_OPS_STAT erts_print(to, to_arg, "atom_put_ops: %ld\n", - erts_smp_atomic_read_nob(&atom_put_ops)); + erts_atomic_read_nob(&atom_put_ops)); #endif if (lock) @@ -138,7 +136,7 @@ atom_hash(Atom* obj) while(len--) { v = *p++; /* latin1 clutch for r16 */ - if ((v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) { + if (len && (v & 0xFE) == 0xC2 && (*p & 0xC0) == 0x80) { v = (v << 6) | (*p & 0x3F); p++; len--; } @@ -246,7 +244,7 @@ erts_atom_put_index(const byte *name, int len, ErtsAtomEncoding enc, int trunc) int aix; #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_inc_nob(&atom_put_ops); + erts_atomic_inc_nob(&atom_put_ops); #endif if (tlen < 0) { @@ -359,32 +357,24 @@ am_atom_put(const char* name, int len) int atom_table_size(void) { int ret; -#ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) atom_read_lock(); -#endif ret = erts_atom_table.entries; -#ifdef ERTS_SMP if (lock) atom_read_unlock(); -#endif return ret; } int atom_table_sz(void) { int ret; -#ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) atom_read_lock(); -#endif ret = index_table_sz(&erts_atom_table); -#ifdef ERTS_SMP if (lock) atom_read_unlock(); -#endif return ret; } @@ -412,19 +402,15 @@ erts_atom_get(const char *name, int len, Eterm* ap, ErtsAtomEncoding enc) void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used) { -#ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) atom_read_lock(); -#endif if (reserved) *reserved = reserved_atom_space; if (used) *used = atom_space; -#ifdef ERTS_SMP if (lock) atom_read_unlock(); -#endif } void @@ -433,16 +419,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT - erts_smp_atomic_init_nob(&atom_put_ops, 0); + erts_atomic_init_nob(&atom_put_ops, 0); #endif - erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + erts_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; @@ -504,4 +491,4 @@ Uint erts_get_atom_limit(void) { return erts_atom_table.limit; -}
\ No newline at end of file +} diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index a44d23b181..fc55b687d4 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -217,6 +217,8 @@ atom discard atom display_items atom dist atom dist_cmd +atom dist_ctrl_put_data +atom dist_data atom Div='/' atom div atom dlink diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 007bf99b6e..00822d1415 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -50,7 +50,7 @@ static struct { Eterm module; - erts_smp_mtx_t mtx; + erts_mtx_t mtx; Export *pending_purge_lambda; Eterm *sprocs; Eterm def_sprocs[10]; @@ -65,12 +65,10 @@ static struct { Process *erts_code_purger = NULL; -#ifdef ERTS_DIRTY_SCHEDULERS Process *erts_dirty_process_code_checker; -#endif -erts_smp_atomic_t erts_copy_literal_area__; +erts_atomic_t erts_copy_literal_area__; #define ERTS_SET_COPY_LITERAL_AREA(LA) \ - erts_smp_atomic_set_nob(&erts_copy_literal_area__, \ + erts_atomic_set_nob(&erts_copy_literal_area__, \ (erts_aint_t) (LA)) Process *erts_literal_area_collector = NULL; @@ -81,7 +79,7 @@ struct ErtsLiteralAreaRef_ { }; struct { - erts_smp_mtx_t mtx; + erts_mtx_t mtx; ErtsLiteralAreaRef *first; ErtsLiteralAreaRef *last; } release_literal_areas; @@ -97,7 +95,8 @@ init_purge_state(void) { purge_state.module = THE_NON_VALUE; - erts_smp_mtx_init(&purge_state.mtx, "purge_state"); + erts_mtx_init(&purge_state.mtx, "purge_state", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); purge_state.pending_purge_lambda = erts_export_put(am_erts_code_purger, am_pending_purge_lambda, 3); @@ -118,10 +117,12 @@ init_purge_state(void) void erts_beam_bif_load_init(void) { - erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); + erts_mtx_init(&release_literal_areas.mtx, "release_literal_areas", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + release_literal_areas.first = NULL; release_literal_areas.last = NULL; - erts_smp_atomic_init_nob(&erts_copy_literal_area__, + erts_atomic_init_nob(&erts_copy_literal_area__, (erts_aint_t) NULL); init_purge_state(); @@ -169,8 +170,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); modp = erts_get_module(mod, erts_active_code_ix()); @@ -194,8 +195,8 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) else { erts_abort_staging_code_ix(); } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; #endif @@ -258,11 +259,10 @@ struct m { Binary* code; Eterm module; Module* modp; - Uint exception; + Eterm exception; }; static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); -#ifdef ERTS_SMP static void smp_code_ix_commiter(void*); static struct /* Protected by code_write_permission */ @@ -270,7 +270,6 @@ static struct /* Protected by code_write_permission */ Process* stager; ErtsThrPrgrLaterOp lop; } committer_state; -#endif static Eterm exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) @@ -279,7 +278,7 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) Eterm res = NIL; while (exceptions > 0) { - if (mp->exception) { + if (is_value(mp->exception)) { res = CONS(hp, mp->module, res); hp += 2; exceptions--; @@ -380,9 +379,9 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { - p[i].exception = 0; + p[i].exception = THE_NON_VALUE; if (p[i].modp->seen) { - p[i].exception = 1; + p[i].exception = am_duplicated; exceptions++; } p[i].modp->seen = 1; @@ -398,8 +397,8 @@ finish_loading_1(BIF_ALIST_1) erts_is_default_trace_enabled() || IF_HIPE(hipe_need_blocking(p[i].modp))) { /* tracing or hipe need thread blocking */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); is_blocking = 1; break; } @@ -416,9 +415,9 @@ finish_loading_1(BIF_ALIST_1) exceptions = 0; for (i = 0; i < n; i++) { - p[i].exception = 0; + p[i].exception = THE_NON_VALUE; if (p[i].modp->curr.code_hdr && p[i].modp->old.code_hdr) { - p[i].exception = 1; + p[i].exception = am_not_purged; exceptions++; } } @@ -439,7 +438,7 @@ finish_loading_1(BIF_ALIST_1) retval = erts_finish_loading(p[i].code, BIF_P, 0, &mod); ASSERT(retval == NIL || retval == am_on_load); if (retval == am_on_load) { - p[i].exception = 1; + p[i].exception = am_on_load; exceptions++; } } @@ -462,9 +461,7 @@ static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, struct m* mods, int nmods, int free_mods) { -#ifdef ERTS_SMP if (is_blocking || !commit) -#endif { if (commit) { int i; @@ -472,7 +469,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, erts_commit_staging_code_ix(); for (i=0; i < nmods; i++) { - if (mods[i].modp->curr.code_hdr) { + if (mods[i].modp->curr.code_hdr + && mods[i].exception != am_on_load) { set_default_trace_pattern(mods[i].module); } #ifdef HIPE @@ -487,13 +485,12 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } erts_release_code_write_permission(); return res; } -#ifdef ERTS_SMP else { ASSERT(is_value(res)); @@ -518,11 +515,9 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, */ ERTS_BIF_YIELD_RETURN(c_p, res); } -#endif } -#ifdef ERTS_SMP static void smp_code_ix_commiter(void* null) { Process* p = committer_state.stager; @@ -532,14 +527,13 @@ static void smp_code_ix_commiter(void* null) committer_state.stager = NULL; #endif erts_release_code_write_permission(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(p)) { erts_resume(p, ERTS_PROC_LOCK_STATUS); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); erts_proc_dec_refc(p); } -#endif /* ERTS_SMP */ @@ -609,9 +603,6 @@ badarg: BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) { -#if !defined(ERTS_DIRTY_SCHEDULERS) - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else Process *rp; int reds = 0; Eterm res; @@ -636,12 +627,11 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls); if (BIF_P != rp) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); ASSERT(is_value(res)); BIF_RET2(res, reds); -#endif } BIF_RETTYPE delete_module_1(BIF_ALIST_1) @@ -679,8 +669,8 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) modp->curr.num_traced_exports > 0 || IF_HIPE(hipe_need_blocking(modp))) { /* tracing or hipe need to go single threaded */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); is_blocking = 1; if (modp->curr.num_breakpoints) { erts_clear_module_break(modp); @@ -697,6 +687,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) Eterm retval; mod.module = BIF_ARG_1; mod.modp = modp; + mod.exception = THE_NON_VALUE; retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); return retval; } @@ -784,16 +775,16 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) /* ToDo: Use code_ix staging instead of thread blocking */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); code_ix = erts_active_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); if (!modp || !modp->on_load || !modp->on_load->code_hdr) { error: - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_ERROR(BIF_P, BADARG); } @@ -832,11 +823,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->beam[1] = 0; } else { if (ep->addressv[code_ix] == ep->beam && - ep->beam[0] == (BeamInstr) em_apply_bif) { + BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -858,14 +849,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } ep->beam[1] = 0; } } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); BIF_RET(am_true); } @@ -926,9 +917,9 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed * any other heap than the message it self. */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { ErlHeapFragment *hf; @@ -1058,10 +1049,8 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed return_ok: -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))) c_p->flags &= ~F_DIRTY_CLA; -#endif return am_ok; @@ -1076,10 +1065,8 @@ literal_gc: *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh, fcalls); -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & F_DIRTY_CLA) return THE_NON_VALUE; -#endif return am_ok; } @@ -1309,7 +1296,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#ifdef ERTS_SMP ErtsThrPrgrLaterOp later_literal_area_switch; @@ -1331,13 +1317,12 @@ static void complete_literal_area_switch(void *literal_area) { Process *p = erts_literal_area_collector; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); erts_resume(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); if (literal_area) erts_release_literal_area((ErtsLiteralArea *) literal_area); } -#endif BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) { @@ -1347,7 +1332,7 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) if (BIF_P != erts_literal_area_collector) BIF_ERROR(BIF_P, EXC_NOTSUP); - erts_smp_mtx_lock(&release_literal_areas.mtx); + erts_mtx_lock(&release_literal_areas.mtx); la_ref = release_literal_areas.first; if (la_ref) { @@ -1356,14 +1341,13 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) release_literal_areas.last = NULL; } - erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_mtx_unlock(&release_literal_areas.mtx); unused_la = ERTS_COPY_LITERAL_AREA(); if (!la_ref) { ERTS_SET_COPY_LITERAL_AREA(NULL); if (unused_la) { -#ifdef ERTS_SMP ErtsLaterReleasLiteralArea *lrlap; lrlap = erts_alloc(ERTS_ALC_T_RELEASE_LAREA, sizeof(ErtsLaterReleasLiteralArea)); @@ -1377,9 +1361,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) + ((unused_la->end - &unused_la->start[0]) - 1)*(sizeof(Eterm)))); -#else - erts_release_literal_area(unused_la); -#endif } BIF_RET(am_false); } @@ -1388,16 +1369,11 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); -#ifdef ERTS_SMP erts_schedule_thr_prgr_later_op(complete_literal_area_switch, unused_la, &later_literal_area_switch); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); -#else - erts_release_literal_area(unused_la); - BIF_RET(am_true); -#endif } @@ -1423,7 +1399,7 @@ erts_purge_state_add_fun(ErlFunEntry *fe) Export * erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe) { - erts_smp_mtx_lock(&purge_state.mtx); + erts_mtx_lock(&purge_state.mtx); if (purge_state.module == fe->module) { /* * The process c_p is about to call a fun in the code @@ -1449,7 +1425,7 @@ erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe) erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); ERTS_VBUMP_ALL_REDS(c_p); } - erts_smp_mtx_unlock(&purge_state.mtx); + erts_mtx_unlock(&purge_state.mtx); return purge_state.pending_purge_lambda; } @@ -1459,9 +1435,9 @@ finalize_purge_operation(Process *c_p, int succeded) Uint ix; if (c_p) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_mtx_lock(&purge_state.mtx); + erts_mtx_lock(&purge_state.mtx); ASSERT(purge_state.module != THE_NON_VALUE); @@ -1477,14 +1453,14 @@ finalize_purge_operation(Process *c_p, int succeded) ERTS_PROC_LOCK_STATUS); if (rp) { erts_resume(rp, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); } } - erts_smp_mtx_unlock(&purge_state.mtx); + erts_mtx_unlock(&purge_state.mtx); if (c_p) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (purge_state.sprocs != &purge_state.def_sprocs[0]) { erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); @@ -1503,7 +1479,6 @@ finalize_purge_operation(Process *c_p, int succeded) purge_state.fe_ix = 0; } -#ifdef ERTS_SMP static ErtsThrPrgrLaterOp purger_lop_data; @@ -1511,9 +1486,9 @@ static void resume_purger(void *unused) { Process *p = erts_code_purger; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); erts_resume(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } static void @@ -1526,7 +1501,6 @@ finalize_purge_abort(void *unused) resume_purger(NULL); } -#endif /* ERTS_SMP */ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) { @@ -1585,9 +1559,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) else { BeamInstr* code; BeamInstr* end; - erts_smp_mtx_lock(&purge_state.mtx); + erts_mtx_lock(&purge_state.mtx); purge_state.module = BIF_ARG_1; - erts_smp_mtx_unlock(&purge_state.mtx); + erts_mtx_unlock(&purge_state.mtx); res = am_true; code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); @@ -1601,9 +1575,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) } } -#ifndef ERTS_SMP - BIF_RET(res); -#else if (res != am_true) BIF_RET(res); else { @@ -1622,7 +1593,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } -#endif } case am_abort: { @@ -1636,11 +1606,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); -#ifndef ERTS_SMP - erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); - finalize_purge_operation(BIF_P, 0); - BIF_RET(am_false); -#else /* * We need to restore the code addresses of the funs in * two stages in order to ensure that we do not get any @@ -1656,7 +1621,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) &purger_lop_data); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_false); -#endif } case am_complete: { @@ -1708,8 +1672,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); if (modp->old.nif) { @@ -1747,8 +1711,8 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_rwunlock_old_code(code_ix); } if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } erts_release_code_write_permission(); @@ -1761,7 +1725,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) sizeof(ErtsLiteralAreaRef)); ref->literal_area = literals; ref->next = NULL; - erts_smp_mtx_lock(&release_literal_areas.mtx); + erts_mtx_lock(&release_literal_areas.mtx); if (release_literal_areas.last) { release_literal_areas.last->next = ref; release_literal_areas.last = ref; @@ -1770,7 +1734,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) release_literal_areas.first = ref; release_literal_areas.last = ref; } - erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_mtx_unlock(&release_literal_areas.mtx); erts_queue_message(erts_literal_area_collector, 0, erts_alloc_message(0, NULL), @@ -1802,22 +1766,23 @@ delete_code(Module* modp) Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { if (ep->addressv[code_ix] == ep->beam) { - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; } - else if (ep->beam[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); DBG_TRACE_MFA_P(&ep->info.mfa, "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler - || !erts_initialized); - } + else { + ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) || + !erts_initialized); + } + } ep->addressv[code_ix] = ep->beam; - ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); ep->beam[1] = 0; DBG_TRACE_MFA_P(&ep->info.mfa, "export invalidation, code_ix=%d", code_ix); diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index b9453c1d9a..c81380c14d 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -46,15 +46,15 @@ #define ReAlloc(P, SIZ) erts_realloc(ERTS_ALC_T_BPD, (P), (SZ)) #define Free(P) erts_free(ERTS_ALC_T_BPD, (P)) -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ +#if defined(ERTS_ENABLE_LOCK_CHECK) +# define ERTS_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ __FILE__, __LINE__) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) #else -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) +# define ERTS_REQ_PROC_MAIN_LOCK(P) +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) #endif #define ERTS_BPF_LOCAL_TRACE 0x01 @@ -73,11 +73,9 @@ extern BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ extern BeamInstr beam_exception_trace[1]; /* OpCode(i_exception_trace) */ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ -erts_smp_atomic32_t erts_active_bp_index; -erts_smp_atomic32_t erts_staging_bp_index; -#ifdef ERTS_DIRTY_SCHEDULERS -erts_smp_mtx_t erts_dirty_bp_ix_mtx; -#endif +erts_atomic32_t erts_active_bp_index; +erts_atomic32_t erts_staging_bp_index; +erts_mtx_t erts_dirty_bp_ix_mtx; /* * Inlined helpers @@ -94,22 +92,18 @@ acquire_bp_sched_ix(Process *c_p) { ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); ASSERT(esdp); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); + erts_mtx_lock(&erts_dirty_bp_ix_mtx); return (Uint32) erts_no_schedulers; } -#endif return (Uint32) esdp->no - 1; } static ERTS_INLINE void release_bp_sched_ix(Uint32 ix) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ix == (Uint32) erts_no_schedulers) - erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); -#endif + erts_mtx_unlock(&erts_dirty_bp_ix_mtx); } @@ -162,11 +156,10 @@ static void bp_hash_delete(bp_time_hash_t *hash); void erts_bp_init(void) { - erts_smp_atomic32_init_nob(&erts_active_bp_index, 0); - erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index"); -#endif + erts_atomic32_init_nob(&erts_active_bp_index, 0); + erts_atomic32_init_nob(&erts_staging_bp_index, 1); + erts_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); } @@ -210,7 +203,7 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) for (fi = 0; fi < num_functions; fi++) { ci = code_hdr->functions[fi]; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (erts_is_function_native(ci)) { continue; } @@ -272,11 +265,11 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) pc = ep->beam; if (ep->addressv[code_ix] == pc) { - if ((*pc == (BeamInstr) em_apply_bif || - *pc == (BeamInstr) em_call_error_handler)) { - continue; + if (BeamIsOpCode(*pc, op_apply_bif) || + BeamIsOpCode(*pc, op_call_error_handler)) { + continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); } else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } @@ -305,7 +298,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local) Uint i; Uint n = f->matched; - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { consolidate_bp_data(fs[i].mod, fs[i].ci, local); @@ -317,7 +310,7 @@ erts_consolidate_bif_bp_data(void) { int i; - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; consolidate_bp_data(0, &ep->info, 0); @@ -373,8 +366,8 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*erts_codeinfo_to_code(ci) != - (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(! BeamIsOpCode(*erts_codeinfo_to_code(ci), + op_i_generic_breakpoint)); } ci->u.gen_bp = NULL; Free(g); @@ -392,17 +385,17 @@ consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) } if (flags & ERTS_BPF_META_TRACE) { dst->meta_tracer = src->meta_tracer; - erts_smp_refc_inc(&dst->meta_tracer->refc, 1); + erts_refc_inc(&dst->meta_tracer->refc, 1); dst->meta_ms = src->meta_ms; MatchSetRef(dst->meta_ms); } if (flags & ERTS_BPF_COUNT) { dst->count = src->count; - erts_smp_refc_inc(&dst->count->refc, 1); + erts_refc_inc(&dst->count->refc, 1); } if (flags & ERTS_BPF_TIME_TRACE) { dst->time = src->time; - erts_smp_refc_inc(&dst->time->refc, 1); + erts_refc_inc(&dst->time->refc, 1); ASSERT(dst->time->hash); } } @@ -413,8 +406,8 @@ erts_commit_staged_bp(void) ErtsBpIndex staging = erts_staging_bp_ix(); ErtsBpIndex active = erts_active_bp_ix(); - erts_smp_atomic32_set_nob(&erts_active_bp_index, staging); - erts_smp_atomic32_set_nob(&erts_staging_bp_index, active); + erts_atomic32_set_nob(&erts_active_bp_index, staging); + erts_atomic32_set_nob(&erts_staging_bp_index, active); } void @@ -422,7 +415,7 @@ erts_install_breakpoints(BpFunctions* f) { Uint i; Uint n = f->matched; - BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); + BeamInstr br = BeamOpCodeAddr(op_i_generic_breakpoint); for (i = 0; i < n; i++) { ErtsCodeInfo* ci = f->matching[i].ci; @@ -467,7 +460,7 @@ static void uninstall_breakpoint(ErtsCodeInfo *ci) { BeamInstr *pc = erts_codeinfo_to_code(ci); - if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + if (BeamIsOpCode(*pc, op_i_generic_breakpoint)) { GenericBp* g = ci->u.gen_bp; if (g->data[erts_active_bp_ix()].flags == 0) { /* @@ -574,7 +567,7 @@ erts_clear_mtrace_bif(ErtsCodeInfo *ci) void erts_clear_debug_break(BpFunctions* f) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); clear_break(f, ERTS_BPF_DEBUG); } @@ -602,7 +595,7 @@ erts_clear_module_break(Module *modp) { Uint n; Uint i; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(modp); code_hdr = modp->curr.code_hdr; if (!code_hdr) { @@ -632,7 +625,7 @@ erts_clear_module_break(Module *modp) { void erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); @@ -649,7 +642,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(info->op, op_i_func_info_IaaI)); g = info->u.gen_bp; bp = &g->data[ix]; @@ -678,12 +671,12 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) if (bp_flags & ERTS_BPF_META_TRACE) { ErtsTracer old_tracer, new_tracer; - old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); + old_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer); new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer); if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { - if (old_tracer == erts_smp_atomic_cmpxchg_acqb( + if (old_tracer == erts_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, (erts_aint_t)new_tracer, (erts_aint_t)old_tracer)) { @@ -695,16 +688,16 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } if (bp_flags & ERTS_BPF_COUNT_ACTIVE) { - erts_smp_atomic_inc_nob(&bp->count->acount); + erts_atomic_inc_nob(&bp->count->acount); } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; - if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || - w == (BeamInstr) BeamOp(op_return_trace) || - w == (BeamInstr) BeamOp(op_i_return_to_trace)) ) { + if (! (BeamIsOpCode(w, op_i_return_time_trace) || + BeamIsOpCode(w, op_return_trace) || + BeamIsOpCode(w, op_i_return_to_trace)) ) { Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { @@ -724,7 +717,7 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } if (bp_flags & ERTS_BPF_DEBUG) { - return (BeamInstr) BeamOp(op_i_debug_breakpoint); + return BeamOpCodeAddr(op_i_debug_breakpoint); } else { return g->orig_instr; } @@ -752,7 +745,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) GenericBpData* bp = NULL; Uint bp_flags = 0; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); g = ep->info.u.gen_bp; if (g) { @@ -776,7 +769,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (bp_flags & ERTS_BPF_META_TRACE) { ErtsTracer old_tracer; - meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); + meta_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer); old_tracer = meta_tracer; flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, 0, &meta_tracer); @@ -784,7 +777,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { ErtsTracer new_tracer = erts_tracer_nil; erts_tracer_update(&new_tracer, meta_tracer); - if (old_tracer == erts_smp_atomic_cmpxchg_acqb( + if (old_tracer == erts_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, (erts_aint_t)new_tracer, (erts_aint_t)old_tracer)) { @@ -911,9 +904,9 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying, } } if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); } } } else { @@ -936,7 +929,7 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying, } } } - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); return result; } @@ -953,12 +946,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, Eterm* E = c_p->stop; w = *c_p->cp; - if (w == (BeamInstr) BeamOp(op_return_trace)) { + if (BeamIsOpCode(w, op_return_trace)) { cpp = &E[2]; - } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { return_to_trace = 1; cpp = &E[0]; - } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { cpp = &E[0]; } else { cpp = NULL; @@ -966,12 +959,12 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, if (cpp) { for (;;) { BeamInstr w = *cp_val(*cpp); - if (w == (BeamInstr) BeamOp(op_return_trace)) { + if (BeamIsOpCode(w, op_return_trace)) { cpp += 3; - } else if (w == (BeamInstr) BeamOp(op_i_return_to_trace)) { + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { return_to_trace = 1; cpp += 1; - } else if (w == (BeamInstr) BeamOp(op_i_return_time_trace)) { + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { cpp += 2; } else { break; @@ -981,9 +974,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, c_p->cp = (BeamInstr *) cp_val(*cpp); ASSERT(is_CP(*cpp)); } - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; } @@ -1023,9 +1016,9 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, the funcinfo is above i. */ c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? beam_exception_trace : beam_return_trace; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } else c_p->stop = E; return tracer; @@ -1042,7 +1035,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) Uint32 six = acquire_bp_sched_ix(c_p); ASSERT(c_p); - ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING + ASSERT(erts_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)); /* get previous timestamp and breakpoint @@ -1123,7 +1116,7 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci) Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); - ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING + ASSERT(erts_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING)); /* get previous timestamp and breakpoint @@ -1205,7 +1198,7 @@ erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, *match_spec_ret = bp->meta_ms; } if (tracer_ret) { - *tracer_ret = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); + *tracer_ret = erts_atomic_read_nob(&bp->meta_tracer->tracer); } return 1; } @@ -1219,7 +1212,7 @@ erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) if (bp) { if (count_ret) { - *count_ret = (Uint) erts_smp_atomic_read_nob(&bp->count->acount); + *count_ret = (Uint) erts_atomic_read_nob(&bp->count->acount); } return 1; } @@ -1300,7 +1293,7 @@ erts_find_local_func(ErtsCodeMFA *mfa) { n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { ci = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); if (mfa->function == ci->mfa.function && mfa->arity == ci->mfa.arity) { @@ -1499,7 +1492,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, Uint common; ErtsBpIndex ix = erts_staging_bp_ix(); - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); g = ci->u.gen_bp; if (g == 0) { int i; @@ -1531,7 +1524,7 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, bp->flags &= ~ERTS_BPF_COUNT_ACTIVE; } else { bp->flags |= ERTS_BPF_COUNT_ACTIVE; - erts_smp_atomic_set_nob(&bp->count->acount, 0); + erts_atomic_set_nob(&bp->count->acount, 0); } ASSERT((bp->flags & ~ERTS_BPF_ALL) == 0); return; @@ -1565,17 +1558,17 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, MatchSetRef(match_spec); bp->meta_ms = match_spec; bmt = Alloc(sizeof(BpMetaTracer)); - erts_smp_refc_init(&bmt->refc, 1); + erts_refc_init(&bmt->refc, 1); erts_tracer_update(&meta_tracer, tracer); /* copy tracer */ - erts_smp_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer); + erts_atomic_init_nob(&bmt->tracer, (erts_aint_t)meta_tracer); bp->meta_tracer = bmt; } else if (break_flags & ERTS_BPF_COUNT) { BpCount* bcp; ASSERT((bp->flags & ERTS_BPF_COUNT) == 0); bcp = Alloc(sizeof(BpCount)); - erts_smp_refc_init(&bcp->refc, 1); - erts_smp_atomic_init_nob(&bcp->acount, 0); + erts_refc_init(&bcp->refc, 1); + erts_atomic_init_nob(&bcp->acount, 0); bp->count = bcp; } else if (break_flags & ERTS_BPF_TIME_TRACE) { BpDataTime* bdt; @@ -1583,12 +1576,8 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); - erts_smp_refc_init(&bdt->refc, 1); -#ifdef ERTS_DIRTY_SCHEDULERS + erts_refc_init(&bdt->refc, 1); bdt->n = erts_no_schedulers + 1; -#else - bdt->n = erts_no_schedulers; -#endif bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); @@ -1620,7 +1609,7 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags) Uint common; ErtsBpIndex ix = erts_staging_bp_ix(); - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); if ((g = ci->u.gen_bp) == NULL) { return 1; @@ -1653,8 +1642,8 @@ clear_function_break(ErtsCodeInfo *ci, Uint break_flags) static void bp_meta_unref(BpMetaTracer* bmt) { - if (erts_smp_refc_dectest(&bmt->refc, 0) <= 0) { - ErtsTracer trc = erts_smp_atomic_read_nob(&bmt->tracer); + if (erts_refc_dectest(&bmt->refc, 0) <= 0) { + ErtsTracer trc = erts_atomic_read_nob(&bmt->tracer); ERTS_TRACER_CLEAR(&trc); Free(bmt); } @@ -1663,7 +1652,7 @@ bp_meta_unref(BpMetaTracer* bmt) static void bp_count_unref(BpCount* bcp) { - if (erts_smp_refc_dectest(&bcp->refc, 0) <= 0) { + if (erts_refc_dectest(&bcp->refc, 0) <= 0) { Free(bcp); } } @@ -1671,7 +1660,7 @@ bp_count_unref(BpCount* bcp) static void bp_time_unref(BpDataTime* bdt) { - if (erts_smp_refc_dectest(&bdt->refc, 0) <= 0) { + if (erts_refc_dectest(&bdt->refc, 0) <= 0) { Uint i = 0; Uint j = 0; Process *h_p = NULL; @@ -1695,7 +1684,7 @@ bp_time_unref(BpDataTime* bdt) if (pbt) { Free(pbt); } - erts_smp_proc_unlock(h_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(h_p, ERTS_PROC_LOCK_MAIN); } } } @@ -1719,7 +1708,7 @@ check_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g = ci->u.gen_bp; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (erts_is_function_native(ci)) { return 0; } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 56fa82b912..a64765822b 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -41,7 +41,7 @@ typedef struct { typedef struct bp_data_time { /* Call time */ Uint n; bp_time_hash_t *hash; - erts_smp_refc_t refc; + erts_refc_t refc; } BpDataTime; typedef struct { @@ -50,13 +50,13 @@ typedef struct { } process_breakpoint_time_t; /* used within psd */ typedef struct { - erts_smp_atomic_t acount; - erts_smp_refc_t refc; + erts_atomic_t acount; + erts_refc_t refc; } BpCount; typedef struct { - erts_smp_atomic_t tracer; - erts_smp_refc_t refc; + erts_atomic_t tracer; + erts_refc_t refc; } BpMetaTracer; typedef struct generic_bp_data { @@ -79,9 +79,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#ifdef ERTS_DIRTY_SCHEDULERS -extern erts_smp_mtx_t erts_dirty_bp_ix_mtx; -#endif +extern erts_mtx_t erts_dirty_bp_ix_mtx; enum erts_break_op{ ERTS_BREAK_NOP = 0, /* Must be false */ @@ -173,17 +171,17 @@ ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -extern erts_smp_atomic32_t erts_active_bp_index; -extern erts_smp_atomic32_t erts_staging_bp_index; +extern erts_atomic32_t erts_active_bp_index; +extern erts_atomic32_t erts_staging_bp_index; ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void) { - return erts_smp_atomic32_read_nob(&erts_active_bp_index); + return erts_atomic32_read_nob(&erts_active_bp_index); } ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void) { - return erts_smp_atomic32_read_nob(&erts_staging_bp_index); + return erts_atomic32_read_nob(&erts_staging_bp_index); } #endif diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index a2060c80de..0f332da63f 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -53,6 +53,8 @@ void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); +static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap); +static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -157,8 +159,8 @@ erts_debug_breakpoint_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_breakpoint_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); erts_bp_match_functions(&f, &mfa, specified); if (boolean == am_true) { @@ -174,8 +176,8 @@ erts_debug_breakpoint_2(BIF_ALIST_2) res = make_small(f.matched); erts_bp_free_matched_functions(&f); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; @@ -197,7 +199,7 @@ void debug_dump_code(BeamInstr *I, int num) erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, i, opc[i].sz-1, code_ptr+1) + 1; break; @@ -317,7 +319,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr); instr = (BeamInstr) code_ptr[0]; for (i = 0; i < NUM_SPECIFIC_OPS; i++) { - if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') { + if (BeamIsOpCode(instr, i) && opc[i].name[0] != '\0') { code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp, i, opc[i].sz-1, code_ptr+1) + 1; break; @@ -424,7 +426,9 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) while (start_prog < prog) { prog--; switch (*prog) { + case 'f': case 'g': + case 'q': *ap++ = *--sp; break; case 'i': /* Initialize packing accumulator. */ @@ -489,6 +493,14 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) case 'n': /* Nil */ erts_print(to, to_arg, "[]"); break; + case 'S': /* Register */ + { + Uint reg_type = (*ap & 1) ? 'y' : 'x'; + Uint n = ap[0] / sizeof(Eterm); + erts_print(to, to_arg, "%c(%d)", reg_type, n); + ap++; + break; + } case 's': /* Any source (tagged constant or register) */ tag = loader_tag(*ap); if (tag == LOADER_X_REG) { @@ -522,12 +534,13 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } ap++; break; - case 'I': /* Untagged integer. */ - case 't': + case 't': /* Untagged integers */ + case 'I': + case 'W': switch (op) { - case op_i_gc_bif1_jIsId: - case op_i_gc_bif2_jIIssd: - case op_i_gc_bif3_jIIssd: + case op_i_gc_bif1_jWstd: + case op_i_gc_bif2_jWtssd: + case op_i_gc_bif3_jWtssd: { const ErtsGcBif* p; BifFunction gcf = (BifFunction) *ap; @@ -549,9 +562,10 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'f': /* Destination label */ { - ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { - erts_print(to, to_arg, "f(" HEXF ")", *ap); + BeamInstr* target = f_to_addr(addr, op, ap); + ErtsCodeMFA* cmfa = find_function_from_pc(target); + if (!cmfa || erts_codemfa_to_code(cmfa) != target) { + erts_print(to, to_arg, "f(" HEXF ")", target); } else { erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, cmfa->function, cmfa->arity); @@ -561,18 +575,18 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'p': /* Pointer (to label) */ { - ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); - if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { - erts_print(to, to_arg, "p(" HEXF ")", *ap); - } else { - erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, - cmfa->function, cmfa->arity); - } + BeamInstr* target = f_to_addr(addr, op, ap); + erts_print(to, to_arg, "p(" HEXF ")", target); ap++; } break; case 'j': /* Pointer (to label) */ - erts_print(to, to_arg, "j(" HEXF ")", *ap); + if (*ap == 0) { + erts_print(to, to_arg, "j(0)"); + } else { + BeamInstr* target = f_to_addr(addr, op, ap); + erts_print(to, to_arg, "j(" HEXF ")", target); + } ap++; break; case 'e': /* Export entry */ @@ -615,12 +629,22 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) unpacked = ap; ap = addr + size; + + /* + * In the code below, never use ap[-1], ap[-2], ... + * (will not work if the arguments have been packed). + * + * Instead use unpacked[-1], unpacked[-2], ... + */ switch (op) { case op_i_select_val_lins_xfI: case op_i_select_val_lins_yfI: + case op_i_select_val_bins_xfI: + case op_i_select_val_bins_yfI: { - int n = ap[-1]; + int n = unpacked[-1]; int ix = n; + Sint32* jump_tab = (Sint32 *)(ap + n); while (ix--) { erts_print(to, to_arg, "%T ", (Eterm) ap[0]); @@ -629,30 +653,19 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } ix = n; while (ix--) { - erts_print(to, to_arg, "f(" HEXF ") ", (Eterm) ap[0]); - ap++; - size++; - } - } - break; - case op_i_select_val_bins_xfI: - case op_i_select_val_bins_yfI: - { - int n = ap[-1]; - - while (n > 0) { - erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]); - ap += 2; - size += 2; - n--; + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } + size += (n+1) / 2; } break; case op_i_select_tuple_arity_xfI: case op_i_select_tuple_arity_yfI: { - int n = ap[-1]; + int n = unpacked[-1]; int ix = n - 1; /* without sentinel */ + Sint32* jump_tab = (Sint32 *)(ap + n); while (ix--) { Uint arity = arityval(ap[0]); @@ -666,39 +679,62 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) size++; ix = n; while (ix--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; + } + size += (n+1) / 2; + } + break; + case op_i_select_val2_xfcc: + case op_i_select_val2_yfcc: + case op_i_select_tuple_arity2_xfAA: + case op_i_select_tuple_arity2_yfAA: + { + Sint32* jump_tab = (Sint32 *) ap; + BeamInstr* target; + int i; + + for (i = 0; i < 2; i++) { + target = f_to_addr_packed(addr, op, jump_tab++); + erts_print(to, to_arg, "f(" HEXF ") ", target); } + size += 1; } break; - case op_i_jump_on_val_xfII: - case op_i_jump_on_val_yfII: + case op_i_jump_on_val_xfIW: + case op_i_jump_on_val_yfIW: { - int n; - for (n = ap[-2]; n > 0; n--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + int n = unpacked[-2]; + Sint32* jump_tab = (Sint32 *) ap; + + size += (n+1) / 2; + while (n-- > 0) { + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } } break; case op_i_jump_on_val_zero_xfI: case op_i_jump_on_val_zero_yfI: { - int n; - for (n = ap[-1]; n > 0; n--) { - erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); - ap++; - size++; + int n = unpacked[-1]; + Sint32* jump_tab = (Sint32 *) ap; + + size += (n+1) / 2; + while (n-- > 0) { + BeamInstr* target = f_to_addr_packed(addr, op, jump_tab); + erts_print(to, to_arg, "f(" HEXF ") ", target); + jump_tab++; } } break; case op_i_put_tuple_xI: case op_i_put_tuple_yI: - case op_new_map_dII: - case op_update_map_assoc_jsdII: - case op_update_map_exact_jsdII: + case op_new_map_dtI: + case op_update_map_assoc_sdtI: + case op_update_map_exact_jsdtI: { int n = unpacked[-1]; @@ -718,6 +754,27 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) } } break; + case op_i_new_small_map_lit_dtq: + { + Eterm *tp = tuple_val(unpacked[-1]); + int n = arityval(*tp); + + while (n > 0) { + switch (loader_tag(ap[0])) { + case LOADER_X_REG: + erts_print(to, to_arg, " x(%d)", loader_x_reg_index(ap[0])); + break; + case LOADER_Y_REG: + erts_print(to, to_arg, " x(%d)", loader_y_reg_index(ap[0])); + break; + default: + erts_print(to, to_arg, " %T", (Eterm) ap[0]); + break; + } + ap++, size++, n--; + } + } + break; case op_i_get_map_elements_fsI: { int n = unpacked[-1]; @@ -766,6 +823,17 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) } } +static BeamInstr* f_to_addr(BeamInstr* base, int op, BeamInstr* ap) +{ + return base - 1 + opc[op].adjust + (Sint32) *ap; +} + +static BeamInstr* f_to_addr_packed(BeamInstr* base, int op, Sint32* ap) +{ + return base - 1 + opc[op].adjust + *ap; +} + + /* * Dirty BIF testing. * @@ -774,10 +842,8 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) * test suite. */ -#ifdef ERTS_DIRTY_SCHEDULERS static int ms_wait(Process *c_p, Eterm etimeout, int busy); static int dirty_send_message(Process *c_p, Eterm to, Eterm tag); -#endif static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I); /* @@ -806,7 +872,6 @@ erts_debug_dirty_io_2(BIF_ALIST_2) BIF_RETTYPE erts_debug_dirty_3(BIF_ALIST_3) { -#ifdef ERTS_DIRTY_SCHEDULERS Eterm argv[2]; switch (BIF_ARG_1) { case am_normal: @@ -836,9 +901,6 @@ erts_debug_dirty_3(BIF_ALIST_3) default: BIF_ERROR(BIF_P, EXC_BADARG); } -#else - BIF_ERROR(BIF_P, EXC_UNDEF); -#endif } @@ -846,7 +908,6 @@ static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) { BIF_RETTYPE ret; -#ifdef ERTS_DIRTY_SCHEDULERS if (am_scheduler == arg1) { ErtsSchedulerData *esdp; if (arg2 != am_type) @@ -1032,13 +1093,9 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) badarg: ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); } -#else - ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); -#endif return ret; } -#ifdef ERTS_DIRTY_SCHEDULERS static int dirty_send_message(Process *c_p, Eterm to, Eterm tag) @@ -1075,7 +1132,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag) if (rp == real_c_p) rp_locks &= ~c_p_locks; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); @@ -1125,13 +1182,8 @@ ms_wait(Process *c_p, Eterm etimeout, int busy) return 1; } -#endif /* ERTS_DIRTY_SCHEDULERS */ -#ifdef ERTS_SMP # define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit()) -#else -# define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit) -#endif /* * The below functions is for testing of the stack diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 79d751d13e..9a77c63390 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -50,45 +50,38 @@ #if defined(NO_JUMP_TABLE) # define OpCase(OpCode) case op_##OpCode # define CountCase(OpCode) case op_count_##OpCode -# define OpCode(OpCode) ((Uint*)op_##OpCode) -# define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;} -# define LabelAddr(Addr) &&##Addr +# define IsOpCode(InstrWord, OpCode) ((InstrWord) == (BeamInstr)op_##OpCode) +# define Goto(Rel) {Go = (Rel); goto emulator_loop;} #else # define OpCase(OpCode) lb_##OpCode # define CountCase(OpCode) lb_count_##OpCode +# define IsOpCode(InstrWord, OpCode) ((InstrWord) == (BeamInstr)&&lb_##OpCode) # define Goto(Rel) goto *((void *)Rel) # define LabelAddr(Label) &&Label -# define OpCode(OpCode) (&&lb_##OpCode) #endif #ifdef ERTS_ENABLE_LOCK_CHECK -# ifdef ERTS_SMP -# define PROCESS_MAIN_CHK_LOCKS(P) \ -do { \ - if ((P)) \ - erts_proc_lc_chk_only_proc_main((P)); \ - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ +# define PROCESS_MAIN_CHK_LOCKS(P) \ +do { \ + if ((P)) \ + erts_proc_lc_chk_only_proc_main((P)); \ + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); \ } while (0) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ -do { \ - if ((P)) \ - erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ - __FILE__, __LINE__); \ +# define ERTS_REQ_PROC_MAIN_LOCK(P) \ +do { \ + if ((P)) \ + erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ + __FILE__, __LINE__); \ } while (0) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ do { \ if ((P)) \ erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN); \ } while (0) -# else -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) -# define PROCESS_MAIN_CHK_LOCKS(P) erts_lc_check_exact(NULL, 0) -# endif #else # define PROCESS_MAIN_CHK_LOCKS(P) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) +# define ERTS_REQ_PROC_MAIN_LOCK(P) +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) #endif /* @@ -113,9 +106,7 @@ do { \ # define CHECK_ARGS(T) #endif -#ifndef MAX -#define MAX(x, y) (((x) > (y)) ? (x) : (y)) -#endif +#define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0) #define GET_BIF_MODULE(p) (p->info.mfa.module) #define GET_BIF_FUNCTION(p) (p->info.mfa.function) @@ -159,50 +150,7 @@ do { \ * Register target (X or Y register). */ -#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb(Target-1) : &xb(Target)) -#define REG_TARGET(Target) (*REG_TARGET_PTR(Target)) - -/* - * Store a result into a register given a destination descriptor. - */ - -#define StoreResult(Result, DestDesc) \ - do { \ - Eterm stb_reg; \ - stb_reg = (DestDesc); \ - CHECK_TERM(Result); \ - REG_TARGET(stb_reg) = (Result); \ - } while (0) - -/* - * Store a result into a register and execute the next instruction. - * Dst points to the word with a destination descriptor, which MUST - * be just before the next instruction. - */ - -#define StoreBifResult(Dst, Result) \ - do { \ - BeamInstr* stb_next; \ - Eterm stb_reg; \ - stb_reg = Arg(Dst); \ - I += (Dst) + 2; \ - stb_next = (BeamInstr *) *I; \ - CHECK_TERM(Result); \ - REG_TARGET(stb_reg) = (Result); \ - Goto(stb_next); \ - } while (0) - -#define ClauseFail() goto jump_f - -#define SAVE_CP(X) \ - do { \ - *(X) = make_cp(c_p->cp); \ - c_p->cp = 0; \ - } while(0) - -#define RESTORE_CP(X) SET_CP(c_p, (BeamInstr *) cp_val(*(X))) - -#define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y)) +#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb((Target)-1) : &xb(Target)) /* * Special Beam instructions. @@ -212,11 +160,6 @@ BeamInstr beam_apply[2]; BeamInstr beam_exit[1]; BeamInstr beam_continue_exit[1]; -BeamInstr* em_call_error_handler; -BeamInstr* em_apply_bif; -BeamInstr* em_call_nif; -BeamInstr* em_call_bif_e; - /* NOTE These should be the only variables containing trace instructions. ** Sometimes tests are form the instruction value, and sometimes @@ -285,13 +228,16 @@ void** beam_ops; HEAP_TOP((P)) = HTOP; \ (P)->stop = E; \ PROCESS_MAIN_CHK_LOCKS((P)); \ - ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P)) + ERTS_UNREQ_PROC_MAIN_LOCK((P)) #define db(N) (N) +#define fb(N) ((Sint)(Sint32)(N)) +#define jb(N) ((Sint)(Sint32)(N)) #define tb(N) (N) #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N))) -#define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) +#define Sb(N) (*REG_TARGET_PTR(N)) +#define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) #define Qb(N) (N) #define Ib(N) (N) #define x(N) reg[N] @@ -299,159 +245,14 @@ void** beam_ops; #define r(N) x(N) /* - * Makes sure that there are StackNeed + HeapNeed + 1 words available - * on the combined heap/stack segment, then allocates StackNeed + 1 - * words on the stack and saves CP. - * - * M is number of live registers to preserve during garbage collection - */ - -#define AH(StackNeed, HeapNeed, M) \ - do { \ - int needed; \ - needed = (StackNeed) + 1; \ - if (E - HTOP < (needed + (HeapNeed))) { \ - SWAPOUT; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \ - reg, (M), FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - SWAPIN; \ - } \ - E -= needed; \ - SAVE_CP(E); \ - } while (0) - -#define Allocate(Ns, Live) AH(Ns, 0, Live) - -#define AllocateZero(Ns, Live) \ - do { Eterm* ptr; \ - int i = (Ns); \ - AH(i, 0, Live); \ - for (ptr = E + i; ptr > E; ptr--) { \ - make_blank(*ptr); \ - } \ - } while (0) - -#define AllocateHeap(Ns, Nh, Live) AH(Ns, Nh, Live) - -#define AllocateHeapZero(Ns, Nh, Live) \ - do { Eterm* ptr; \ - int i = (Ns); \ - AH(i, Nh, Live); \ - for (ptr = E + i; ptr > E; ptr--) { \ - make_blank(*ptr); \ - } \ - } while (0) - -#define AllocateInit(Ns, Live, Y) \ - do { AH(Ns, 0, Live); make_blank(Y); } while (0) - -/* - * Like the AH macro, but allocates no additional heap space. - */ - -#define A(StackNeed, M) AH(StackNeed, 0, M) - -#define D(N) \ - RESTORE_CP(E); \ - E += (N) + 1; - - - -#define TestBinVHeap(VNh, Nh, Live) \ - do { \ - unsigned need = (Nh); \ - if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ - SWAPOUT; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - - - -/* - * Check if Nh words of heap are available; if not, do a garbage collection. - * Live is number of active argument registers to be preserved. - */ - -#define TestHeap(Nh, Live) \ - do { \ - unsigned need = (Nh); \ - if (E - HTOP < need) { \ - SWAPOUT; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - -/* - * Check if Nh words of heap are available; if not, do a garbage collection. - * Live is number of active argument registers to be preserved. - * Takes special care to preserve Extra if a garbage collection occurs. - */ - -#define TestHeapPreserve(Nh, Live, Extra) \ - do { \ - unsigned need = (Nh); \ - if (E - HTOP < need) { \ - SWAPOUT; \ - reg[Live] = Extra; \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \ - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ - PROCESS_MAIN_CHK_LOCKS(c_p); \ - Extra = reg[Live]; \ - SWAPIN; \ - } \ - HEAP_SPACE_VERIFIED(need); \ - } while (0) - -#define TestHeapPutList(Need, Reg) \ - do { \ - TestHeap((Need), 1); \ - PutList(Reg, r(0), r(0)); \ - CHECK_TERM(r(0)); \ - } while (0) - -#define Init(N) make_blank(yb(N)) - -#define Init2(Y1, Y2) do { make_blank(Y1); make_blank(Y2); } while (0) -#define Init3(Y1, Y2, Y3) \ - do { make_blank(Y1); make_blank(Y2); make_blank(Y3); } while (0) - -#define MakeFun(FunP, NumFree) \ - do { \ - HEAVY_SWAPOUT; \ - r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ - HEAVY_SWAPIN; \ - } while (0) - -#define PutTuple(Dst, Arity) \ - do { \ - Dst = make_tuple(HTOP); \ - pt_arity = (Arity); \ - } while (0) - -/* * Check that we haven't used the reductions and jump to function pointed to by * the I register. If we are out of reductions, do a context switch. */ #define DispatchMacro() \ do { \ - BeamInstr* dis_next; \ - dis_next = (BeamInstr *) *I; \ + BeamInstr dis_next; \ + dis_next = *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -459,12 +260,12 @@ void** beam_ops; } else { \ goto context_switch; \ } \ - } while (0) + } while (0) \ #define DispatchMacroFun() \ do { \ - BeamInstr* dis_next; \ - dis_next = (BeamInstr *) *I; \ + BeamInstr dis_next; \ + dis_next = *I; \ CHECK_ARGS(I); \ if (FCALLS > 0 || FCALLS > neg_o_reds) { \ FCALLS--; \ @@ -474,23 +275,23 @@ void** beam_ops; } \ } while (0) -#define DispatchMacrox() \ - do { \ - if (FCALLS > 0) { \ - Eterm* dis_next; \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - dis_next = (Eterm *) *I; \ - FCALLS--; \ - CHECK_ARGS(I); \ - Goto(dis_next); \ - } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ - && FCALLS > neg_o_reds) { \ - goto save_calls1; \ - } else { \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - CHECK_ARGS(I); \ - goto context_switch; \ - } \ +#define DispatchMacrox() \ + do { \ + if (FCALLS > 0) { \ + BeamInstr dis_next; \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ + dis_next = *I; \ + FCALLS--; \ + CHECK_ARGS(I); \ + Goto(dis_next); \ + } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ + && FCALLS > neg_o_reds) { \ + goto save_calls1; \ + } else { \ + SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ + CHECK_ARGS(I); \ + goto context_switch; \ + } \ } while (0) #ifdef DEBUG @@ -509,20 +310,7 @@ void** beam_ops; # define Dispatchfun() DispatchMacroFun() #endif -#define Self(R) R = c_p->common.id -#define Node(R) R = erts_this_node->sysname - #define Arg(N) I[(N)+1] -#define Next(N) \ - I += (N) + 1; \ - ASSERT(VALID_INSTR(*I)); \ - Goto(*I) - -#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0) -#define NextPF(N, Dst) \ - I += N + 1; \ - ASSERT(VALID_INSTR(Dst)); \ - Goto(Dst) #define GetR(pos, tr) \ do { \ @@ -539,97 +327,20 @@ void** beam_ops; CHECK_TERM(tr); \ } while (0) -#define GetArg1(N, Dst) GetR((N), Dst) - -#define GetArg2(N, Dst1, Dst2) \ - do { \ - GetR(N, Dst1); \ - GetR((N)+1, Dst2); \ - } while (0) - -#define PutList(H, T, Dst) \ - do { \ - HTOP[0] = (H); HTOP[1] = (T); \ - Dst = make_list(HTOP); \ - HTOP += 2; \ - } while (0) - -#define Swap(R1, R2) \ - do { \ - Eterm V = R1; \ - R1 = R2; \ - R2 = V; \ - } while (0) - -#define SwapTemp(R1, R2, Tmp) \ - do { \ - Eterm V = R1; \ - R1 = R2; \ - R2 = Tmp = V; \ - } while (0) - -#define Move(Src, Dst) Dst = (Src) - -#define Move2Par(S1, D1, S2, D2) \ - do { \ - Eterm V1, V2; \ - V1 = (S1); V2 = (S2); D1 = V1; D2 = V2; \ - } while (0) - -#define MoveShift(Src, SD, D) \ - do { \ - Eterm V; \ - V = Src; D = SD; SD = V; \ - } while (0) - -#define MoveDup(Src, D1, D2) \ - do { \ - D1 = D2 = (Src); \ - } while (0) - -#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3) - -#define MoveWindow3(S1, S2, S3, D) \ - do { \ - Eterm xt0, xt1, xt2; \ - Eterm *y = &D; \ - xt0 = S1; \ - xt1 = S2; \ - xt2 = S3; \ - y[0] = xt0; \ - y[1] = xt1; \ - y[2] = xt2; \ - } while (0) - -#define MoveWindow4(S1, S2, S3, S4, D) \ - do { \ - Eterm xt0, xt1, xt2, xt3; \ - Eterm *y = &D; \ - xt0 = S1; \ - xt1 = S2; \ - xt2 = S3; \ - xt3 = S4; \ - y[0] = xt0; \ - y[1] = xt1; \ - y[2] = xt2; \ - y[3] = xt3; \ - } while (0) - -#define MoveWindow5(S1, S2, S3, S4, S5, D) \ - do { \ - Eterm xt0, xt1, xt2, xt3, xt4; \ - Eterm *y = &D; \ - xt0 = S1; \ - xt1 = S2; \ - xt2 = S3; \ - xt3 = S4; \ - xt4 = S5; \ - y[0] = xt0; \ - y[1] = xt1; \ - y[2] = xt2; \ - y[3] = xt3; \ - y[4] = xt4; \ - } while (0) +#define PUT_TERM_REG(term, desc) \ +do { \ + switch (loader_tag(desc)) { \ + case LOADER_X_REG: \ + x(loader_x_reg_index(desc)) = (term); \ + break; \ + case LOADER_Y_REG: \ + y(loader_y_reg_index(desc)) = (term); \ + break; \ + default: \ + ASSERT(0); \ + break; \ + } \ +} while(0) #define DispatchReturn \ do { \ @@ -644,409 +355,14 @@ do { \ } \ } while (0) -#define MoveReturn(Src) \ - x(0) = (Src); \ - I = c_p->cp; \ - ASSERT(VALID_INSTR(*c_p->cp)); \ - c_p->cp = 0; \ - CHECK_TERM(r(0)); \ - DispatchReturn - -#define DeallocateReturn(Deallocate) \ - do { \ - int words_to_pop = (Deallocate); \ - SET_I((BeamInstr *) cp_val(*E)); \ - E = ADD_BYTE_OFFSET(E, words_to_pop); \ - CHECK_TERM(r(0)); \ - DispatchReturn; \ - } while (0) - -#define MoveDeallocateReturn(Src, Deallocate) \ - x(0) = (Src); \ - DeallocateReturn(Deallocate) - -#define MoveCall(Src, CallDest, Size) \ - x(0) = (Src); \ - SET_CP(c_p, I+Size+1); \ - SET_I((BeamInstr *) CallDest); \ - Dispatch(); - -#define MoveCallLast(Src, CallDest, Deallocate) \ - x(0) = (Src); \ - RESTORE_CP(E); \ - E = ADD_BYTE_OFFSET(E, (Deallocate)); \ - SET_I((BeamInstr *) CallDest); \ - Dispatch(); - -#define MoveCallOnly(Src, CallDest) \ - x(0) = (Src); \ - SET_I((BeamInstr *) CallDest); \ - Dispatch(); - -#define MoveJump(Src) \ - r(0) = (Src); \ - SET_I((BeamInstr *) Arg(0)); \ - Goto(*I); - -#define GetList(Src, H, T) \ - do { \ - Eterm* tmp_ptr = list_val(Src); \ - Eterm hd, tl; \ - hd = CAR(tmp_ptr); \ - tl = CDR(tmp_ptr); \ - H = hd; T = tl; \ - } while (0) - -#define GetTupleElement(Src, Element, Dest) \ - do { \ - Eterm* src; \ - src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ - (Dest) = *src; \ - } while (0) - -#define GetTupleElement2(Src, Element, Dest) \ - do { \ - Eterm* src; \ - Eterm* dst; \ - Eterm E1, E2; \ - src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ - dst = &(Dest); \ - E1 = src[0]; \ - E2 = src[1]; \ - dst[0] = E1; \ - dst[1] = E2; \ - } while (0) - -#define GetTupleElement2Y(Src, Element, D1, D2) \ - do { \ - Eterm* src; \ - Eterm E1, E2; \ - src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ - E1 = src[0]; \ - E2 = src[1]; \ - D1 = E1; \ - D2 = E2; \ - } while (0) - -#define GetTupleElement3(Src, Element, Dest) \ - do { \ - Eterm* src; \ - Eterm* dst; \ - Eterm E1, E2, E3; \ - src = ADD_BYTE_OFFSET(tuple_val(Src), (Element)); \ - dst = &(Dest); \ - E1 = src[0]; \ - E2 = src[1]; \ - E3 = src[2]; \ - dst[0] = E1; \ - dst[1] = E2; \ - dst[2] = E3; \ - } while (0) - -#define EqualImmed(X, Y, Action) if (X != Y) { Action; } -#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } -#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; } -#define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; } -#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action) -#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action) -#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action) -#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action) - -#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } - -#define IsInteger(Src, Fail) if (is_not_integer(Src)) { Fail; } - -#define IsNumber(X, Fail) if (is_not_integer(X) && is_not_float(X)) { Fail; } - -#define IsAtom(Src, Fail) if (is_not_atom(Src)) { Fail; } - -#define IsIntegerAllocate(Src, Need, Alive, Fail) \ - if (is_not_integer(Src)) { Fail; } \ - A(Need, Alive) - -#define IsNil(Src, Fail) if (is_not_nil(Src)) { Fail; } - -#define IsList(Src, Fail) if (is_not_list(Src) && is_not_nil(Src)) { Fail; } - -#define IsNonemptyList(Src, Fail) if (is_not_list(Src)) { Fail; } - -#define IsNonemptyListAllocate(Src, Need, Alive, Fail) \ - if (is_not_list(Src)) { Fail; } \ - A(Need, Alive) - -#define IsNonemptyListTestHeap(Need, Alive, Fail) \ - if (is_not_list(x(0))) { Fail; } \ - TestHeap(Need, Alive) - -#define IsNonemptyListGetList(Src, H, T, Fail) \ - if (is_not_list(Src)) { \ - Fail; \ - } else { \ - Eterm* tmp_ptr = list_val(Src); \ - Eterm hd, tl; \ - hd = CAR(tmp_ptr); \ - tl = CDR(tmp_ptr); \ - H = hd; T = tl; \ - } - -#define IsTuple(X, Action) if (is_not_tuple(X)) Action - -#define IsArity(Pointer, Arity, Fail) \ - if (*tuple_val(Pointer) != (Arity)) { \ - Fail; \ - } - -#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; } - -#define GetMapElement(Src, Key, Dst, Fail) \ - do { \ - Eterm _res = get_map_element(Src, Key); \ - if (is_non_value(_res)) { \ - Fail; \ - } \ - Dst = _res; \ - } while (0) - -#define GetMapElementHash(Src, Key, Hx, Dst, Fail) \ - do { \ - Eterm _res = get_map_element_hash(Src, Key, Hx); \ - if (is_non_value(_res)) { \ - Fail; \ - } \ - Dst = _res; \ - } while (0) - -#define IsFunction(X, Action) \ - do { \ - if ( !(is_any_fun(X)) ) { \ - Action; \ - } \ - } while (0) - -#define IsFunction2(F, A, Action) \ - do { \ - if (erl_is_function(c_p, F, A) != am_true ) { \ - Action; \ - } \ - } while (0) - #ifdef DEBUG -#define IsTupleOfArity(Src, Arityval, Fail) \ - do { \ - if (!(is_tuple(Src) && *tuple_val(Src) == Arityval)) { \ - Fail; \ - } \ - } while (0) -#else -#define IsTupleOfArity(Src, Arityval, Fail) \ - do { \ - if (!(is_boxed(Src) && *tuple_val(Src) == Arityval)) { \ - Fail; \ - } \ - } while (0) -#endif - -#define IsTaggedTuple(Src,Arityval,Tag,Fail) \ - do { \ - if (!(is_tuple(Src) && \ - (tuple_val(Src))[0] == Arityval && \ - (tuple_val(Src))[1] == Tag)) { \ - Fail; \ - } \ - } while (0) - -#define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; } - -#define IsBinary(Src, Fail) \ - if (is_not_binary(Src) || binary_bitsize(Src) != 0) { Fail; } - -#define IsBitstring(Src, Fail) \ - if (is_not_binary(Src)) { Fail; } - -#if defined(ARCH_64) -#define BsSafeMul(A, B, Fail, Target) \ - do { Uint64 _res = (A) * (B); \ - if (_res / B != A) { Fail; } \ - Target = _res; \ - } while (0) +/* Better static type testing by the C compiler */ +# define BEAM_IS_TUPLE(Src) is_tuple(Src) #else -#define BsSafeMul(A, B, Fail, Target) \ - do { Uint64 _res = (Uint64)(A) * (Uint64)(B); \ - if ((_res >> (8*sizeof(Uint))) != 0) { Fail; } \ - Target = _res; \ - } while (0) +/* Better performance */ +# define BEAM_IS_TUPLE(Src) is_boxed(Src) #endif -#define BsGetFieldSize(Bits, Unit, Fail, Target) \ - do { \ - Sint _signed_size; Uint _uint_size; \ - Uint temp_bits; \ - if (is_small(Bits)) { \ - _signed_size = signed_val(Bits); \ - if (_signed_size < 0) { Fail; } \ - _uint_size = (Uint) _signed_size; \ - } else { \ - if (!term_to_Uint(Bits, &temp_bits)) { Fail; } \ - _uint_size = temp_bits; \ - } \ - BsSafeMul(_uint_size, Unit, Fail, Target); \ - } while (0) - -#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target) \ - do { \ - Sint _signed_size; Uint _uint_size; \ - Uint temp_bits; \ - if (is_small(Bits)) { \ - _signed_size = signed_val(Bits); \ - if (_signed_size < 0) { Fail; } \ - _uint_size = (Uint) _signed_size; \ - } else { \ - if (!term_to_Uint(Bits, &temp_bits)) { Fail; } \ - _uint_size = (Uint) temp_bits; \ - } \ - Target = _uint_size * Unit; \ - } while (0) - -#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; Sint _size; \ - if (!is_small(Sz) || (_size = unsigned_val(Sz)) > 64) { Fail; } \ - _size *= ((Flags) >> 3); \ - TestHeap(FLOAT_SIZE_OBJECT, Live); \ - _mb = ms_matchbuffer(Ms); \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - if (is_non_value(_result)) { Fail; } \ - else { Dst = _result; } \ - } while (0) - -#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; \ - TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), Live); \ - _mb = ms_matchbuffer(Ms); \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - if (is_non_value(_result)) { Fail; } \ - else { Dst = _result; } \ - } while (0) - -#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; Uint _size; \ - BsGetFieldSize(Sz, ((Flags) >> 3), Fail, _size); \ - TestHeap(ERL_SUB_BIN_SIZE, Live); \ - _mb = ms_matchbuffer(Ms); \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - if (is_non_value(_result)) { Fail; } \ - else { Dst = _result; } \ - } while (0) - -#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - Eterm _result; \ - TestHeap(ERL_SUB_BIN_SIZE, Live); \ - _mb = ms_matchbuffer(Ms); \ - if (((_mb->size - _mb->offset) % Unit) == 0) { \ - LIGHT_SWAPOUT; \ - _result = erts_bs_get_binary_all_2(c_p, _mb); \ - LIGHT_SWAPIN; \ - HEAP_SPACE_VERIFIED(0); \ - ASSERT(is_value(_result)); \ - Dst = _result; \ - } else { \ - HEAP_SPACE_VERIFIED(0); \ - Fail; } \ - } while (0) - -#define BsSkipBits2(Ms, Bits, Unit, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - size_t new_offset; \ - Uint _size; \ - _mb = ms_matchbuffer(Ms); \ - BsGetFieldSize(Bits, Unit, Fail, _size); \ - new_offset = _mb->offset + _size; \ - if (new_offset <= _mb->size) { _mb->offset = new_offset; } \ - else { Fail; } \ - } while (0) - -#define BsSkipBitsAll2(Ms, Unit, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - _mb = ms_matchbuffer(Ms); \ - if (((_mb->size - _mb->offset) % Unit) == 0) {_mb->offset = _mb->size; } \ - else { Fail; } \ - } while (0) - -#define BsSkipBitsImm2(Ms, Bits, Fail) \ - do { \ - ErlBinMatchBuffer *_mb; \ - size_t new_offset; \ - _mb = ms_matchbuffer(Ms); \ - new_offset = _mb->offset + (Bits); \ - if (new_offset <= _mb->size) { _mb->offset = new_offset; } \ - else { Fail; } \ - } while (0) - -#define NewBsPutIntegerImm(Sz, Flags, Src) \ - do { \ - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), (Sz), (Flags)))) { goto badarg; } \ - } while (0) - -#define NewBsPutInteger(Sz, Flags, Src) \ - do { \ - Sint _size; \ - BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size); \ - if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), _size, (Flags)))) \ - { goto badarg; } \ - } while (0) - -#define NewBsPutFloatImm(Sz, Flags, Src) \ - do { \ - if (!erts_new_bs_put_float(c_p, (Src), (Sz), (Flags))) { goto badarg; } \ - } while (0) - -#define NewBsPutFloat(Sz, Flags, Src) \ - do { \ - Sint _size; \ - BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size); \ - if (!erts_new_bs_put_float(c_p, (Src), _size, (Flags))) { goto badarg; } \ - } while (0) - -#define NewBsPutBinary(Sz, Flags, Src) \ - do { \ - Sint _size; \ - BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size); \ - if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), _size))) { goto badarg; } \ - } while (0) - -#define NewBsPutBinaryImm(Sz, Src) \ - do { \ - if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), (Sz)))) { goto badarg; } \ - } while (0) - -#define NewBsPutBinaryAll(Src, Unit) \ - do { \ - if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2((Src), (Unit)))) { goto badarg; } \ - } while (0) - - -#define IsPort(Src, Fail) if (is_not_port(Src)) { Fail; } -#define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; } -#define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; } - /* * process_main() is already huge, so we want to avoid inlining * into it. Especially functions that are seldom used. @@ -1062,6 +378,7 @@ do { \ * The following functions are called directly by process_main(). * Don't inline them. */ +static void init_emulator_finish(void) NOINLINE; static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE; static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, @@ -1070,20 +387,21 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, BeamInstr *I, Uint offs) NOINLINE; -static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg, - BeamInstr *I, Uint offs) NOINLINE; +static BeamInstr* apply(Process* p, Eterm* reg, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) NOINLINE; static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) NOINLINE; -static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE; -static Eterm update_map_assoc(Process* p, Eterm* reg, - Eterm map, BeamInstr* I) NOINLINE; -static Eterm update_map_exact(Process* p, Eterm* reg, - Eterm map, BeamInstr* I) NOINLINE; +static Eterm new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) NOINLINE; +static Eterm new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, + Uint live, BeamInstr* ptr) NOINLINE; +static Eterm update_map_assoc(Process* p, Eterm* reg, Uint live, + Uint n, BeamInstr* new_p) NOINLINE; +static Eterm update_map_exact(Process* p, Eterm* reg, Uint live, + Uint n, Eterm* new_p) NOINLINE; static Eterm get_map_element(Eterm map, Eterm key); static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx); @@ -1115,6 +433,12 @@ init_emulator(void) # define REG_stop asm("%l3") # define REG_I asm("%l4") # define REG_fcalls asm("%l5") +#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG) +# define REG_xregs asm("%r12") +# define REG_htop +# define REG_stop asm("%r13") +# define REG_I asm("%rbx") +# define REG_fcalls asm("%r14") #else # define REG_xregs # define REG_htop @@ -1234,6 +558,13 @@ init_emulator(void) #define ERTS_DBG_CHK_REDS(P, FC) #endif +#ifdef NO_FPE_SIGNALS +# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT +# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR +#else +# define ERTS_NO_FPE_CHECK_INIT(p) +# define ERTS_NO_FPE_ERROR(p, a, b) +#endif /* * process_main() is called twice: @@ -1293,12 +624,10 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #ifndef NO_JUMP_TABLE static void* opcodes[] = { DEFINE_OPCODES }; #else - int Go; + register BeamInstr Go; #endif #endif - Eterm pt_arity; /* Used by do_put_tuple */ - Uint64 start_time = 0; /* Monitor long schedule */ BeamInstr* start_time_i = NULL; @@ -1315,7 +644,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) * Note: c_p->arity must be set to reflect the number of useful terms in * c_p->arg_reg before calling the scheduler. */ - if (!init_done) { + if (ERTS_UNLIKELY(!init_done)) { /* This should only be reached during the init phase when only the main * process is running. I.e. there is no race for init_done. */ @@ -1348,16 +677,16 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) } PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = erts_schedule(NULL, c_p, reds_used); ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); start_time = 0; #ifdef DEBUG - pid = c_p->common.id; /* Save for debugging purpouses */ + pid = c_p->common.id; /* Save for debugging purposes */ #endif - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_MSACC_UPDATE_CACHE_X(); @@ -1371,7 +700,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { int reds; Eterm* argp; - BeamInstr *next; + BeamInstr next; int i; argp = c_p->arg_reg; @@ -1403,7 +732,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) ERTS_DBG_CHK_REDS(c_p, FCALLS); - next = (BeamInstr *) *I; + next = *I; SWAPIN; ASSERT(VALID_INSTR(next)); @@ -1439,1923 +768,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #ifdef NO_JUMP_TABLE switch (Go) { #endif -#include "beam_hot.h" - - { - Eterm increment_reg_val; - Eterm increment_val; - Uint live; - Eterm result; - - OpCase(i_increment_rIId): - increment_reg_val = x(0); - I--; - goto do_increment; - - OpCase(i_increment_xIId): - increment_reg_val = xb(Arg(0)); - goto do_increment; - - OpCase(i_increment_yIId): - increment_reg_val = yb(Arg(0)); - goto do_increment; - - do_increment: - increment_val = Arg(1); - if (is_small(increment_reg_val)) { - Sint i = signed_val(increment_reg_val) + increment_val; - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(3, result); - } - } - - live = Arg(2); - HEAVY_SWAPOUT; - reg[live] = increment_reg_val; - reg[live+1] = make_small(increment_val); - result = erts_gc_mixed_plus(c_p, reg, live); - HEAVY_SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - StoreBifResult(3, result); - } - ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); - goto find_func_info; - } - -#define DO_OUTLINED_ARITH_2(name, Op1, Op2) \ - do { \ - Eterm result; \ - Uint live = Arg(1); \ - \ - HEAVY_SWAPOUT; \ - reg[live] = Op1; \ - reg[live+1] = Op2; \ - result = erts_gc_##name(c_p, reg, live); \ - HEAVY_SWAPIN; \ - ERTS_HOLE_CHECK(c_p); \ - if (is_value(result)) { \ - StoreBifResult(4, result); \ - } \ - goto lb_Cl_error; \ - } while (0) - - { - Eterm PlusOp1, PlusOp2; - Eterm result; - - OpCase(i_plus_jIxxd): - PlusOp1 = xb(Arg(2)); - PlusOp2 = xb(Arg(3)); - goto do_plus; - - OpCase(i_plus_jIxyd): - PlusOp1 = xb(Arg(2)); - PlusOp2 = yb(Arg(3)); - goto do_plus; - - OpCase(i_plus_jIssd): - GetArg2(2, PlusOp1, PlusOp2); - goto do_plus; - - do_plus: - if (is_both_small(PlusOp1, PlusOp2)) { - Sint i = signed_val(PlusOp1) + signed_val(PlusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(mixed_plus, PlusOp1, PlusOp2); - } - - { - Eterm MinusOp1, MinusOp2; - Eterm result; - - OpCase(i_minus_jIxxd): - MinusOp1 = xb(Arg(2)); - MinusOp2 = xb(Arg(3)); - goto do_minus; - - OpCase(i_minus_jIssd): - GetArg2(2, MinusOp1, MinusOp2); - goto do_minus; - - do_minus: - if (is_both_small(MinusOp1, MinusOp2)) { - Sint i = signed_val(MinusOp1) - signed_val(MinusOp2); - ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); - if (MY_IS_SSMALL(i)) { - result = make_small(i); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(mixed_minus, MinusOp1, MinusOp2); - } - - { - Eterm is_eq_exact_lit_val; - - OpCase(i_is_eq_exact_literal_fxc): - is_eq_exact_lit_val = xb(Arg(1)); - goto do_is_eq_exact_literal; - - OpCase(i_is_eq_exact_literal_fyc): - is_eq_exact_lit_val = yb(Arg(1)); - goto do_is_eq_exact_literal; - - do_is_eq_exact_literal: - if (!eq(Arg(2), is_eq_exact_lit_val)) { - ClauseFail(); - } - Next(3); - } - - { - Eterm is_ne_exact_lit_val; - - OpCase(i_is_ne_exact_literal_fxc): - is_ne_exact_lit_val = xb(Arg(1)); - goto do_is_ne_exact_literal; - - OpCase(i_is_ne_exact_literal_fyc): - is_ne_exact_lit_val = yb(Arg(1)); - goto do_is_ne_exact_literal; - - do_is_ne_exact_literal: - if (eq(Arg(2), is_ne_exact_lit_val)) { - ClauseFail(); - } - Next(3); - } - - OpCase(i_move_call_only_fc): { - r(0) = Arg(1); - } - /* FALL THROUGH */ - OpCase(i_call_only_f): { - SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); - } - - OpCase(i_move_call_last_fPc): { - r(0) = Arg(2); - } - /* FALL THROUGH */ - OpCase(i_call_last_fP): { - RESTORE_CP(E); - E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); - } - - OpCase(i_move_call_cf): { - r(0) = Arg(0); - I++; - } - /* FALL THROUGH */ - OpCase(i_call_f): { - SET_CP(c_p, I+2); - SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); - } - - OpCase(i_move_call_ext_last_ePc): { - r(0) = Arg(2); - } - /* FALL THROUGH */ - OpCase(i_call_ext_last_eP): - RESTORE_CP(E); - E = ADD_BYTE_OFFSET(E, Arg(1)); - - /* - * Note: The pointer to the export entry is never NULL; if the module - * is not loaded, it points to code which will invoke the error handler - * (see lb_call_error_handler below). - */ - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(i_move_call_ext_ce): { - r(0) = Arg(0); - I++; - } - /* FALL THROUGH */ - OpCase(i_call_ext_e): - SET_CP(c_p, I+2); - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(i_move_call_ext_only_ec): { - r(0) = Arg(1); - } - /* FALL THROUGH */ - OpCase(i_call_ext_only_e): - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0)); - Dispatchx(); - - OpCase(init_y): { - BeamInstr *next; - - PreFetch(1, next); - make_blank(yb(Arg(0))); - NextPF(1, next); - } - - OpCase(i_trim_I): { - BeamInstr *next; - Uint words; - Uint cp; - - words = Arg(0); - cp = E[0]; - PreFetch(1, next); - E += words; - E[0] = cp; - NextPF(1, next); - } - - OpCase(move_x1_c): { - x(1) = Arg(0); - Next(1); - } - - OpCase(move_x2_c): { - x(2) = Arg(0); - Next(1); - } - - OpCase(return): { - SET_I(c_p->cp); - DTRACE_RETURN_FROM_PC(c_p); - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; - CHECK_TERM(r(0)); - HEAP_SPACE_VERIFIED(0); - DispatchReturn; - } - - /* - * Send is almost a standard call-BIF with two arguments, except for: - * 1) It cannot be traced. - * 2) There is no pointer to the send_2 function stored in - * the instruction. - */ - - OpCase(send): { - BeamInstr *next; - Eterm result; - - if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - c_p->arity = 2; - c_p->current = NULL; - goto context_switch3; - } - - PRE_BIF_SWAPOUT(c_p); - c_p->fcalls = FCALLS - 1; - result = erl_send(c_p, r(0), x(1)); - PreFetch(0, next); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - HTOP = HEAP_TOP(c_p); - FCALLS = c_p->fcalls; - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(0, next); - } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+1); - SET_I(c_p->i); - SWAPIN; - Dispatch(); - } - goto find_func_info; - } - - { - Eterm element_index; - Eterm element_tuple; - - OpCase(i_element_jxsd): - element_tuple = xb(Arg(1)); - goto do_element; - - OpCase(i_element_jysd): - element_tuple = yb(Arg(1)); - goto do_element; - - do_element: - GetArg1(2, element_index); - if (is_small(element_index) && is_tuple(element_tuple)) { - Eterm* tp = tuple_val(element_tuple); - - if ((signed_val(element_index) >= 1) && - (signed_val(element_index) <= arityval(*tp))) { - Eterm result = tp[signed_val(element_index)]; - StoreBifResult(3, result); - } - } - } - /* Fall through */ - - OpCase(badarg_j): - badarg: - c_p->freason = BADARG; - goto lb_Cl_error; - - { - Eterm fast_element_tuple; - - OpCase(i_fast_element_jxId): - fast_element_tuple = xb(Arg(1)); - goto do_fast_element; - - OpCase(i_fast_element_jyId): - fast_element_tuple = yb(Arg(1)); - goto do_fast_element; - - do_fast_element: - if (is_tuple(fast_element_tuple)) { - Eterm* tp = tuple_val(fast_element_tuple); - Eterm pos = Arg(2); /* Untagged integer >= 1 */ - if (pos <= arityval(*tp)) { - Eterm result = tp[pos]; - StoreBifResult(3, result); - } - } - goto badarg; - } - - OpCase(catch_yf): - c_p->catches++; - yb(Arg(0)) = Arg(1); - Next(2); - - OpCase(catch_end_y): { - c_p->catches--; - make_blank(yb(Arg(0))); - if (is_non_value(r(0))) { - c_p->fvalue = NIL; - if (x(1) == am_throw) { - r(0) = x(2); - } else { - if (x(1) == am_error) { - SWAPOUT; - x(2) = add_stacktrace(c_p, x(2), x(3)); - SWAPIN; - } - /* only x(2) is included in the rootset here */ - if (E - HTOP < 3) { - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - } - r(0) = TUPLE2(HTOP, am_EXIT, x(2)); - HTOP += 3; - } - } - CHECK_TERM(r(0)); - Next(1); - } - - OpCase(try_end_y): { - c_p->catches--; - make_blank(yb(Arg(0))); - if (is_non_value(r(0))) { - c_p->fvalue = NIL; - r(0) = x(1); - x(1) = x(2); - x(2) = x(3); - } - Next(1); - } - - /* - * Skeleton for receive statement: - * - * recv_mark L1 Optional - * call make_ref/monitor Optional - * ... - * recv_set L1 Optional - * L1: <-------------------+ - * <-----------+ | - * | | - * loop_rec L2 ------+---+ | - * ... | | | - * remove_message | | | - * jump L3 | | | - * ... | | | - * loop_rec_end L1 --+ | | - * L2: <---------------+ | - * wait L1 -----------------+ or wait_timeout - * timeout - * - * L3: Code after receive... - * - * - */ - - OpCase(recv_mark_f): { - /* - * Save the current position in message buffer and the - * the label for the loop_rec/2 instruction for the - * the receive statement. - */ - c_p->msg.mark = (BeamInstr *) Arg(0); - c_p->msg.saved_last = c_p->msg.last; - Next(1); - } - - OpCase(i_recv_set): { - /* - * If the mark is valid (points to the loop_rec/2 - * instruction that follows), we know that the saved - * position points to the first message that could - * possibly be matched out. - * - * If the mark is invalid, we do nothing, meaning that - * we will look through all messages in the message queue. - */ - if (c_p->msg.mark == (BeamInstr *) (I+1)) { - c_p->msg.save = c_p->msg.saved_last; - } - I++; - /* Fall through to the loop_rec/2 instruction */ - } - - /* - * Pick up the next message and place it in x(0). - * If no message, jump to a wait or wait_timeout instruction. - */ - OpCase(i_loop_rec_f): - { - BeamInstr *next; - ErtsMessage* msgp; - - /* - * We need to disable GC while matching messages - * in the queue. This since messages with data outside - * the heap will be corrupted by a GC. - */ - ASSERT(!(c_p->flags & F_DELAY_GC)); - c_p->flags |= F_DELAY_GC; - - loop_rec__: - - PROCESS_MAIN_CHK_LOCKS(c_p); - - msgp = PEEK_MESSAGE(c_p); - - if (!msgp) { -#ifdef ERTS_SMP - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Make sure messages wont pass exit signals... */ - if (ERTS_PROC_PENDING_EXIT(c_p)) { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - SWAPOUT; - c_p->flags &= ~F_DELAY_GC; - c_p->arity = 0; - goto do_schedule; /* Will be rescheduled for exit */ - } - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - msgp = PEEK_MESSAGE(c_p); - if (msgp) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - else -#endif - { - c_p->flags &= ~F_DELAY_GC; - SET_I((BeamInstr *) Arg(0)); - Goto(*I); /* Jump to a wait or wait_timeout instruction */ - } - } - if (is_non_value(ERL_MESSAGE_TERM(msgp))) { - SWAPOUT; /* erts_decode_dist_message() may write to heap... */ - if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { - /* - * A corrupt distribution message that we weren't able to decode; - * remove it... - */ - /* No swapin should be needed */ - ASSERT(HTOP == c_p->htop && E == c_p->stop); - /* TODO: Add DTrace probe for this bad message situation? */ - UNLINK_MESSAGE(c_p, msgp); - msgp->next = NULL; - erts_cleanup_messages(msgp); - goto loop_rec__; - } - SWAPIN; - } - PreFetch(1, next); - r(0) = ERL_MESSAGE_TERM(msgp); - NextPF(1, next); - } - - /* - * Remove a (matched) message from the message queue. - */ - OpCase(remove_message): { - BeamInstr *next; - ErtsMessage* msgp; - PROCESS_MAIN_CHK_LOCKS(c_p); - - ERTS_CHK_MBUF_SZ(c_p); - - PreFetch(0, next); - msgp = PEEK_MESSAGE(c_p); - - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_receive); - } - if (ERL_MESSAGE_TOKEN(msgp) == NIL) { -#ifdef USE_VM_PROBES - if (DT_UTAG(c_p) != NIL) { - if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { - SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; - } else { - DT_UTAG(c_p) = NIL; - SEQ_TRACE_TOKEN(c_p) = NIL; - } - } else { -#endif - SEQ_TRACE_TOKEN(c_p) = NIL; -#ifdef USE_VM_PROBES - } - DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; -#endif - } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { - Eterm msg; - SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); -#ifdef USE_VM_PROBES - if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { - if (DT_UTAG(c_p) == NIL) { - DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); - } - DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; - } else { -#endif - ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); - ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); - ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); - ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); - ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); - c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { - c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); - } - msg = ERL_MESSAGE_TERM(msgp); - seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, - c_p->common.id, c_p); -#ifdef USE_VM_PROBES - } -#endif - } -#ifdef USE_VM_PROBES - if (DTRACE_ENABLED(message_receive)) { - Eterm token2 = NIL; - DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); - Sint tok_label = 0; - Sint tok_lastcnt = 0; - Sint tok_serial = 0; - - dtrace_proc_str(c_p, receiver_name); - token2 = SEQ_TRACE_TOKEN(c_p); - if (have_seqtrace(token2)) { - tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); - tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); - tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); - } - DTRACE6(message_receive, - receiver_name, size_object(ERL_MESSAGE_TERM(msgp)), - c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial); - } -#endif - UNLINK_MESSAGE(c_p, msgp); - JOIN_MESSAGE(c_p); - CANCEL_TIMER(c_p); - - erts_save_message_in_proc(c_p, msgp); - c_p->flags &= ~F_DELAY_GC; - - if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { - /* - * We want to GC soon but we leave a few - * reductions giving the message some time - * to turn into garbage. - */ - ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); - } - - ERTS_DBG_CHK_REDS(c_p, FCALLS); - ERTS_CHK_MBUF_SZ(c_p); - - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - NextPF(0, next); - } - - /* - * Advance the save pointer to the next message (the current - * message didn't match), then jump to the loop_rec instruction. - */ - OpCase(loop_rec_end_f): { - - ASSERT(c_p->flags & F_DELAY_GC); - - SET_I((BeamInstr *) Arg(0)); - SAVE_MESSAGE(c_p); - if (FCALLS > 0 || FCALLS > neg_o_reds) { - FCALLS--; - goto loop_rec__; - } - - c_p->flags &= ~F_DELAY_GC; - c_p->i = I; - SWAPOUT; - c_p->arity = 0; - c_p->current = NULL; - goto do_schedule; - } - /* - * Prepare to wait for a message or a timeout, whichever occurs first. - * - * Note: In order to keep the compatibility between 32 and 64 bits - * emulators, only timeout values that can be represented in 32 bits - * (unsigned) or less are allowed. - */ - - - OpCase(i_wait_timeout_fs): { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - - /* Fall through */ - } - OpCase(i_wait_timeout_locked_fs): { - Eterm timeout_value; - - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if (c_p->flags & (F_INSLPQUEUE | F_TIMO)) { - goto wait2; - } - GetArg1(1, timeout_value); - if (timeout_value != make_small(0)) { - - if (timeout_value == am_infinity) - c_p->flags |= F_TIMO; - else { - int tres = erts_set_proc_timer_term(c_p, timeout_value); - if (tres == 0) { - /* - * The timer routiner will set c_p->i to the value in - * c_p->def_arg_reg[0]. Note that it is safe to use this - * location because there are no living x registers in - * a receive statement. - */ - BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; - *pi = I+3; - } - else { /* Wrong time */ - OpCase(i_wait_error_locked): { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - /* Fall through */ - } - OpCase(i_wait_error): { - c_p->freason = EXC_TIMEOUT_VALUE; - goto find_func_info; - } - } - } - - /* - * Prepare to wait indefinitely for a new message to arrive - * (or the time set above if falling through from above). - * - * When a new message arrives, control will be transferred - * the loop_rec instruction (at label L1). In case of - * of timeout, control will be transferred to the timeout - * instruction following the wait_timeout instruction. - */ - - OpCase(wait_locked_f): - OpCase(wait_f): - - wait2: { -#ifndef ERTS_SMP - if (ERTS_PROC_IS_EXITING(c_p)) { - /* - * I non smp case: - * - * Currently executing process might be sent an exit - * signal if it is traced by a port that it also is - * linked to, and the port terminates during the - * trace. In this case we do *not* want to clear - * the active flag, which will make the process hang - * in limbo forever. - */ - SWAPOUT; - c_p->arity = 0; - goto do_schedule; - } -#endif - c_p->i = (BeamInstr *) Arg(0); /* L1 */ - SWAPOUT; - c_p->arity = 0; - - if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) - erts_smp_atomic32_read_band_relb(&c_p->state, - ~ERTS_PSFLG_ACTIVE); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - c_p->current = NULL; - goto do_schedule; - } - OpCase(wait_unlocked_f): { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - goto wait2; - } - } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - Next(2); - } - - OpCase(i_wait_timeout_fI): { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - } - - OpCase(i_wait_timeout_locked_fI): - { - /* - * If we have already set the timer, we must NOT set it again. Therefore, - * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. - */ - if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { - BeamInstr** p = (BeamInstr **) c_p->def_arg_reg; - *p = I+3; - erts_set_proc_timer_uword(c_p, Arg(1)); - } - goto wait2; - } - - /* - * A timeout has occurred. Reset the save pointer so that the next - * receive statement will examine the first message first. - */ - OpCase(timeout_locked): { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); - } - - OpCase(timeout): { - BeamInstr *next; - - PreFetch(0, next); - if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { - trace_receive(c_p, am_clock_service, am_timeout, NULL); - } - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { - save_calls(c_p, &exp_timeout); - } - c_p->flags &= ~F_TIMO; - JOIN_MESSAGE(c_p); - NextPF(0, next); - } - - { - Eterm select_val2; - - OpCase(i_select_tuple_arity2_yfAAff): - select_val2 = yb(Arg(0)); - goto do_select_tuple_arity2; - - OpCase(i_select_tuple_arity2_xfAAff): - select_val2 = xb(Arg(0)); - goto do_select_tuple_arity2; - - do_select_tuple_arity2: - if (is_not_tuple(select_val2)) { - goto select_val2_fail; - } - select_val2 = *tuple_val(select_val2); - goto do_select_val2; - - OpCase(i_select_val2_yfccff): - select_val2 = yb(Arg(0)); - goto do_select_val2; - - OpCase(i_select_val2_xfccff): - select_val2 = xb(Arg(0)); - goto do_select_val2; - - do_select_val2: - if (select_val2 == Arg(2)) { - I += 3; - } else if (select_val2 == Arg(3)) { - I += 4; - } - - select_val2_fail: - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - { - Eterm select_val; - - OpCase(i_select_tuple_arity_xfI): - select_val = xb(Arg(0)); - goto do_select_tuple_arity; - - OpCase(i_select_tuple_arity_yfI): - select_val = yb(Arg(0)); - goto do_select_tuple_arity; - - do_select_tuple_arity: - if (is_tuple(select_val)) { - select_val = *tuple_val(select_val); - goto do_linear_search; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - - OpCase(i_select_val_lins_xfI): - select_val = xb(Arg(0)); - goto do_linear_search; - - OpCase(i_select_val_lins_yfI): - select_val = yb(Arg(0)); - goto do_linear_search; - - do_linear_search: { - BeamInstr *vs = &Arg(3); - int ix = 0; - - for(;;) { - if (vs[ix+0] >= select_val) { ix += 0; break; } - if (vs[ix+1] >= select_val) { ix += 1; break; } - ix += 2; - } - - if (vs[ix] == select_val) { - I += ix + Arg(2) + 2; - } - - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - OpCase(i_select_val_bins_xfI): - select_val = xb(Arg(0)); - goto do_binary_search; - - OpCase(i_select_val_bins_yfI): - select_val = yb(Arg(0)); - goto do_binary_search; - - do_binary_search: - { - struct Pairs { - BeamInstr val; - BeamInstr* addr; - }; - struct Pairs* low; - struct Pairs* high; - struct Pairs* mid; - int bdiff; /* int not long because the arrays aren't that large */ - - low = (struct Pairs *) &Arg(3); - high = low + Arg(2); - - /* The pointer subtraction (high-low) below must produce - * a signed result, because high could be < low. That - * requires the compiler to insert quite a bit of code. - * - * However, high will be > low so the result will be - * positive. We can use that knowledge to optimise the - * entire sequence, from the initial comparison to the - * computation of mid. - * - * -- Mikael Pettersson, Acumem AB - * - * Original loop control code: - * - * while (low < high) { - * mid = low + (high-low) / 2; - * - */ - while ((bdiff = (int)((char*)high - (char*)low)) > 0) { - unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); - - mid = (struct Pairs*)((char*)low + boffset); - if (select_val < mid->val) { - high = mid; - } else if (select_val > mid->val) { - low = mid + 1; - } else { - SET_I(mid->addr); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - } - - { - Eterm jump_on_val_zero_index; - - OpCase(i_jump_on_val_zero_yfI): - jump_on_val_zero_index = yb(Arg(0)); - goto do_jump_on_val_zero_index; - - OpCase(i_jump_on_val_zero_xfI): - jump_on_val_zero_index = xb(Arg(0)); - goto do_jump_on_val_zero_index; - - do_jump_on_val_zero_index: - if (is_small(jump_on_val_zero_index)) { - jump_on_val_zero_index = signed_val(jump_on_val_zero_index); - if (jump_on_val_zero_index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - { - Eterm jump_on_val_index; - - - OpCase(i_jump_on_val_yfII): - jump_on_val_index = yb(Arg(0)); - goto do_jump_on_val_index; - - OpCase(i_jump_on_val_xfII): - jump_on_val_index = xb(Arg(0)); - goto do_jump_on_val_index; - - do_jump_on_val_index: - if (is_small(jump_on_val_index)) { - jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); - if (jump_on_val_index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]); - Goto(*I); - } - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - do_put_tuple: { - Eterm* hp = HTOP; - - *hp++ = make_arityval(pt_arity); - - do { - Eterm term = *I++; - switch (loader_tag(term)) { - case LOADER_X_REG: - *hp++ = x(loader_x_reg_index(term)); - break; - case LOADER_Y_REG: - *hp++ = y(loader_y_reg_index(term)); - break; - default: - *hp++ = term; - break; - } - } while (--pt_arity != 0); - HTOP = hp; - Goto(*I); - } - - OpCase(new_map_dII): { - Eterm res; - - HEAVY_SWAPOUT; - res = new_map(c_p, reg, I-1); - HEAVY_SWAPIN; - StoreResult(res, Arg(0)); - Next(3+Arg(2)); - } - -#define PUT_TERM_REG(term, desc) \ -do { \ - switch (loader_tag(desc)) { \ - case LOADER_X_REG: \ - x(loader_x_reg_index(desc)) = (term); \ - break; \ - case LOADER_Y_REG: \ - y(loader_y_reg_index(desc)) = (term); \ - break; \ - default: \ - ASSERT(0); \ - break; \ - } \ -} while(0) - - OpCase(i_get_map_elements_fsI): { - Eterm map; - BeamInstr *fs; - Uint sz, n; - - GetArg1(1, map); - - /* this instruction assumes Arg1 is a map, - * i.e. that it follows a test is_map if needed. - */ - - n = (Uint)Arg(2) / 3; - fs = &Arg(3); /* pattern fields and target registers */ - - if (is_flatmap(map)) { - flatmap_t *mp; - Eterm *ks; - Eterm *vs; - - mp = (flatmap_t *)flatmap_val(map); - sz = flatmap_get_size(mp); - - if (sz == 0) { - ClauseFail(); - } - - ks = flatmap_get_keys(mp); - vs = flatmap_get_values(mp); - - while(sz) { - if (EQ((Eterm) fs[0], *ks)) { - PUT_TERM_REG(*vs, fs[1]); - n--; - fs += 3; - /* no more values to fetch, we are done */ - if (n == 0) { - I = fs; - Next(-1); - } - } - ks++, sz--, vs++; - } - - ClauseFail(); - } else { - const Eterm *v; - Uint32 hx; - ASSERT(is_hashmap(map)); - while(n--) { - hx = fs[2]; - ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); - if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { - ClauseFail(); - } - PUT_TERM_REG(*v, fs[1]); - fs += 3; - } - I = fs; - Next(-1); - } - } -#undef PUT_TERM_REG - - OpCase(update_map_assoc_jsdII): { - Eterm res; - Eterm map; - - GetArg1(1, map); - HEAVY_SWAPOUT; - res = update_map_assoc(c_p, reg, map, I); - HEAVY_SWAPIN; - if (is_value(res)) { - StoreResult(res, Arg(2)); - Next(5+Arg(4)); - } else { - /* - * This can only happen if the code was compiled - * with the compiler in OTP 17. - */ - c_p->freason = BADMAP; - c_p->fvalue = map; - goto lb_Cl_error; - } - } - - OpCase(update_map_exact_jsdII): { - Eterm res; - Eterm map; - - GetArg1(1, map); - HEAVY_SWAPOUT; - res = update_map_exact(c_p, reg, map, I); - HEAVY_SWAPIN; - if (is_value(res)) { - StoreResult(res, Arg(2)); - Next(5+Arg(4)); - } else { - goto lb_Cl_error; - } - } - - - /* - * All guards with zero arguments have special instructions: - * self/0 - * node/0 - * - * All other guard BIFs take one or two arguments. - */ - - /* - * Guard BIF in head. On failure, ignore the error and jump - * to the code for the next clause. We don't support tracing - * of guard BIFs. - */ - - OpCase(bif1_fbsd): - { - ErtsBifFunc bf; - Eterm tmp_reg[1]; - Eterm result; - - GetArg1(2, tmp_reg[0]); - bf = (BifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(3, result); - } - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - - /* - * Guard BIF in body. It can fail like any BIF. No trace support. - */ - - OpCase(bif1_body_bsd): - { - ErtsBifFunc bf; - - Eterm tmp_reg[1]; - Eterm result; - - GetArg1(1, tmp_reg[0]); - bf = (ErtsBifFunc) Arg(0); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(2, result); - } - reg[0] = tmp_reg[0]; - SWAPOUT; - I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif1_jIsId): - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(3); - - GetArg1(2, x(live)); - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(4, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - x(0) = x(live); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif2_jIIssd): /* Note, one less parameter than the i_gc_bif1 - and i_gc_bif3 */ - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(2); - - GetArg2(3, x(live), x(live+1)); - /* - * XXX This calling convention does not make sense. 'live' - * should point out the first argument, not the second - * (i.e. 'live' should not be incremented below). - */ - live++; - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(5, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - live--; - x(0) = x(live); - x(1) = x(live+1); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - OpCase(i_gc_bif3_jIIssd): - { - typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); - GcBifFunction bf; - Eterm result; - Uint live = (Uint) Arg(2); - - x(live) = x(SCRATCH_X_REG); - GetArg2(3, x(live+1), x(live+2)); - /* - * XXX This calling convention does not make sense. 'live' - * should point out the first argument, not the third - * (i.e. 'live' should not be incremented below). - */ - live += 2; - bf = (GcBifFunction) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, live); - ERTS_CHK_MBUF_SZ(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - SWAPIN; - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(5, result); - } - if (Arg(0) != 0) { - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - live -= 2; - x(0) = x(live); - x(1) = x(live+1); - x(2) = x(live+2); - I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); - goto post_error_handling; - } - - /* - * Guards bifs and, or, xor in guards. - */ - OpCase(i_bif2_fbssd): - { - Eterm tmp_reg[2]; - ErtsBifFunc bf; - Eterm result; - - GetArg2(2, tmp_reg[0], tmp_reg[1]); - bf = (ErtsBifFunc) Arg(1); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS; - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(result)) { - StoreBifResult(4, result); - } - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - - /* - * Guards bifs and, or, xor, relational operators in body. - */ - OpCase(i_bif2_body_bssd): - { - Eterm tmp_reg[2]; - ErtsBifFunc bf; - Eterm result; - - GetArg2(1, tmp_reg[0], tmp_reg[1]); - bf = (ErtsBifFunc) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_HOLE_CHECK(c_p); - if (is_value(result)) { - ASSERT(!is_CP(result)); - StoreBifResult(3, result); - } - reg[0] = tmp_reg[0]; - reg[1] = tmp_reg[1]; - SWAPOUT; - I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); - goto post_error_handling; - } - - /* - * The most general BIF call. The BIF may build any amount of data - * on the heap. The result is always returned in r(0). - */ - OpCase(call_bif_e): - { - ErtsBifFunc bf; - Eterm result; - BeamInstr *next; - ErlHeapFragment *live_hf_end; - Export *export = (Export*)Arg(0); - - - if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - c_p->arity = GET_BIF_ARITY(export); - c_p->current = &export->info.mfa; - goto context_switch3; - } - - ERTS_MSACC_SET_BIF_STATE_CACHED_X( - GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - - bf = GET_BIF_ADDRESS(export); - - PRE_BIF_SWAPOUT(c_p); - ERTS_DBG_CHK_REDS(c_p, FCALLS); - c_p->fcalls = FCALLS - 1; - if (FCALLS <= 0) { - save_calls(c_p, export); - } - PreFetch(1, next); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_HOLE_CHECK(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); - result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); - E = c_p->stop; - } - PROCESS_MAIN_CHK_LOCKS(c_p); - HTOP = HEAP_TOP(c_p); - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - /* We have to update the cache if we are enabled in order - to make sure no book keeping is done after we disabled - msacc. We don't always do this as it is quite expensive. */ - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) - ERTS_MSACC_UPDATE_CACHE_X(); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (is_value(result)) { - r(0) = result; - CHECK_TERM(r(0)); - NextPF(1, next); - } else if (c_p->freason == TRAP) { - SET_CP(c_p, I+2); - SET_I(c_p->i); - SWAPIN; - Dispatch(); - } - - /* - * Error handling. SWAPOUT is not needed because it was done above. - */ - ASSERT(c_p->stop == E); - I = handle_error(c_p, I, reg, &export->info.mfa); - goto post_error_handling; - } - - /* - * Arithmetic operations. - */ - - OpCase(i_times_jIssd): - { - Eterm Op1, Op2; - GetArg2(2, Op1, Op2); - DO_OUTLINED_ARITH_2(mixed_times, Op1, Op2); - } - - OpCase(i_m_div_jIssd): - { - Eterm Op1, Op2; - GetArg2(2, Op1, Op2); - DO_OUTLINED_ARITH_2(mixed_div, Op1, Op2); - } - - OpCase(i_int_div_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (Op2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(Op1, Op2)) { - Sint ires = signed_val(Op1) / signed_val(Op2); - if (MY_IS_SSMALL(ires)) { - Eterm result = make_small(ires); - StoreBifResult(4, result); - } - } - DO_OUTLINED_ARITH_2(int_div, Op1, Op2); - } - - { - Eterm RemOp1, RemOp2; - - OpCase(i_rem_jIxxd): - RemOp1 = xb(Arg(2)); - RemOp2 = xb(Arg(3)); - goto do_rem; - - OpCase(i_rem_jIssd): - GetArg2(2, RemOp1, RemOp2); - goto do_rem; - - do_rem: - if (RemOp2 == SMALL_ZERO) { - goto badarith; - } else if (is_both_small(RemOp1, RemOp2)) { - Eterm result = make_small(signed_val(RemOp1) % signed_val(RemOp2)); - StoreBifResult(4, result); - } else { - DO_OUTLINED_ARITH_2(int_rem, RemOp1, RemOp2); - } - } - - { - Eterm BandOp1, BandOp2; - - OpCase(i_band_jIxcd): - BandOp1 = xb(Arg(2)); - BandOp2 = Arg(3); - goto do_band; - - OpCase(i_band_jIssd): - GetArg2(2, BandOp1, BandOp2); - goto do_band; - - do_band: - if (is_both_small(BandOp1, BandOp2)) { - /* - * No need to untag -- TAG & TAG == TAG. - */ - Eterm result = BandOp1 & BandOp2; - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(band, BandOp1, BandOp2); - } - - /* - * An error occurred in an arithmetic operation or test that could - * appear either in a head or in a body. - * In a head, execution should continue at failure address in Arg(0). - * In a body, Arg(0) == 0 and an exception should be raised. - */ - lb_Cl_error: { - if (Arg(0) != 0) { - OpCase(jump_f): { - jump_f: - SET_I((BeamInstr *) Arg(0)); - Goto(*I); - } - } - ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); - goto find_func_info; - } - - OpCase(i_bor_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (is_both_small(Op1, Op2)) { - /* - * No need to untag -- TAG | TAG == TAG. - */ - Eterm result = Op1 | Op2; - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(bor, Op1, Op2); - } - - OpCase(i_bxor_jIssd): - { - Eterm Op1, Op2; - - GetArg2(2, Op1, Op2); - if (is_both_small(Op1, Op2)) { - /* - * TAG ^ TAG == 0. - * - * Therefore, we perform the XOR operation on the tagged values, - * and OR in the tag bits. - */ - Eterm result = (Op1 ^ Op2) | make_small(0); - StoreBifResult(4, result); - } - DO_OUTLINED_ARITH_2(bxor, Op1, Op2); - } - - { - Eterm Op1, Op2; - Sint i; - Sint ires; - Eterm* bigp; - Eterm tmp_big[2]; - - OpCase(i_bsr_jIssd): - GetArg2(2, Op1, Op2); - if (is_small(Op2)) { - i = -signed_val(Op2); - if (is_small(Op1)) { - goto small_shift; - } else if (is_big(Op1)) { - if (i == 0) { - StoreBifResult(4, Op1); - } - ires = big_size(Op1); - goto big_shift; - } - } else if (is_big(Op2)) { - /* - * N bsr NegativeBigNum == N bsl MAX_SMALL - * N bsr PositiveBigNum == N bsl MIN_SMALL - */ - Op2 = make_small(bignum_header_is_neg(*big_val(Op2)) ? - MAX_SMALL : MIN_SMALL); - goto do_bsl; - } - goto badarith; - - OpCase(i_bsl_jIssd): - GetArg2(2, Op1, Op2); - do_bsl: - if (is_small(Op2)) { - i = signed_val(Op2); - - if (is_small(Op1)) { - small_shift: - ires = signed_val(Op1); - - if (i == 0 || ires == 0) { - StoreBifResult(4, Op1); - } else if (i < 0) { /* Right shift */ - i = -i; - if (i >= SMALL_BITS-1) { - Op1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO; - } else { - Op1 = make_small(ires >> i); - } - StoreBifResult(4, Op1); - } else if (i < SMALL_BITS-1) { /* Left shift */ - if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) || - ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) { - Op1 = make_small(ires << i); - StoreBifResult(4, Op1); - } - } - ires = 1; /* big_size(small_to_big(Op1)) */ - - big_shift: - if (i > 0) { /* Left shift. */ - ires += (i / D_EXP); - } else { /* Right shift. */ - if (ires <= (-i / D_EXP)) - ires = 3; /* ??? */ - else - ires -= (-i / D_EXP); - } - { - ires = BIG_NEED_SIZE(ires+1); - /* - * Slightly conservative check the size to avoid - * allocating huge amounts of memory for bignums that - * clearly would overflow the arity in the header - * word. - */ - if (ires-8 > BIG_ARITY_MAX) { - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - TestHeapPreserve(ires+1, Arg(1), Op1); - if (is_small(Op1)) { - Op1 = small_to_big(signed_val(Op1), tmp_big); - } - bigp = HTOP; - Op1 = big_lshift(Op1, i, bigp); - if (is_big(Op1)) { - HTOP += bignum_header_arity(*HTOP) + 1; - } - HEAP_SPACE_VERIFIED(0); - if (is_nil(Op1)) { - /* - * This result must have been only slight larger - * than allowed since it wasn't caught by the - * previous test. - */ - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - ERTS_HOLE_CHECK(c_p); - StoreBifResult(4, Op1); - } - } else if (is_big(Op1)) { - if (i == 0) { - StoreBifResult(4, Op1); - } - ires = big_size(Op1); - goto big_shift; - } - } else if (is_big(Op2)) { - if (bignum_header_is_neg(*big_val(Op2))) { - /* - * N bsl NegativeBigNum is either 0 or -1, depending on - * the sign of N. Since we don't believe this case - * is common, do the calculation with the minimum - * amount of code. - */ - Op2 = make_small(MIN_SMALL); - goto do_bsl; - } else if (is_small(Op1) || is_big(Op1)) { - /* - * N bsl PositiveBigNum is too large to represent. - */ - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - } - /* Fall through if the left argument is not an integer. */ - } - /* - * One or more non-integer arguments. - */ - goto badarith; - } - - OpCase(i_int_bnot_jsId): - { - Eterm bnot_val; - - GetArg1(1, bnot_val); - if (is_small(bnot_val)) { - bnot_val = make_small(~signed_val(bnot_val)); - } else { - Uint live = Arg(2); - HEAVY_SWAPOUT; - reg[live] = bnot_val; - bnot_val = erts_gc_bnot(c_p, reg, live); - HEAVY_SWAPIN; - ERTS_HOLE_CHECK(c_p); - if (is_nil(bnot_val)) { - goto lb_Cl_error; - } - } - StoreBifResult(3, bnot_val); - } - - badarith: - c_p->freason = BADARITH; - goto lb_Cl_error; - - OpCase(i_apply): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+1); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_last_P): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0)); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(0)); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_only): { - BeamInstr *next; - HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg, I, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(apply_I): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0), NULL, 0); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+2); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(apply_last_IP): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0), I, Arg(1)); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I(next); - Dispatch(); - } - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); - goto post_error_handling; - } - - OpCase(i_apply_fun): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+1); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_apply_fun_last_P): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(0)); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_apply_fun_only): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = apply_fun(c_p, r(0), x(1), reg); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - - OpCase(i_call_fun_I): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, I+2); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - OpCase(i_call_fun_last_IP): { - BeamInstr *next; - - HEAVY_SWAPOUT; - next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_CP(c_p, (BeamInstr *) E[0]); - E = ADD_BYTE_OFFSET(E, Arg(1)); - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } +#include "beam_hot.h" #ifdef DEBUG /* @@ -3399,7 +813,7 @@ do { \ Eterm* argp; int i; - if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) { + if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) { c_p->i = beam_exit; c_p->arity = 0; c_p->current = NULL; @@ -3456,64 +870,25 @@ do { \ goto do_schedule1; } - OpCase(set_tuple_element_sdP): { - Eterm element; - Eterm tuple; - BeamInstr *next; - Eterm* p; - - PreFetch(3, next); - GetArg1(0, element); - tuple = REG_TARGET(Arg(1)); - ASSERT(is_tuple(tuple)); - p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2)); - *p = element; - NextPF(3, next); - } +#include "beam_warm.h" OpCase(normal_exit): { SWAPOUT; c_p->freason = EXC_NORMAL; - c_p->arity = 0; /* In case this process will never be garbed again. */ - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + c_p->arity = 0; /* In case this process will ever be garbed again. */ + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); erts_do_exit_process(c_p, am_normal); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); goto do_schedule; } OpCase(continue_exit): { - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); erts_continue_exit_process(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); goto do_schedule; } - OpCase(i_raise): { - Eterm raise_trace = x(2); - Eterm raise_value = x(1); - struct StackTrace *s; - - c_p->fvalue = raise_value; - c_p->ftrace = raise_trace; - s = get_trace_from_exc(raise_trace); - if (s == NULL) { - c_p->freason = EXC_ERROR; - } else { - c_p->freason = PRIMARY_EXCEPTION(s->freason); - } - goto find_func_info; - } - - { - Eterm badmatch_val; - - OpCase(badmatch_x): - badmatch_val = xb(Arg(0)); - c_p->fvalue = badmatch_val; - c_p->freason = BADMATCH; - } - /* Fall through here */ - find_func_info: { SWAPOUT; I = handle_error(c_p, I, reg, NULL); @@ -3554,194 +929,6 @@ do { \ } } - { - Eterm nif_bif_result; - Eterm bif_nif_arity; - - OpCase(call_nif): - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF - * - * This layout is determined by the NifExport struct - */ - BifFunction vbf; - ErlHeapFragment *live_hf_end; - ErtsCodeMFA *codemfa; - - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - - codemfa = erts_code_to_codemfa(I); - - c_p->current = codemfa; /* current and vbf set to please handle_error */ - - DTRACE_NIF_ENTRY(c_p, codemfa); - - HEAVY_SWAPOUT; - - PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = codemfa->arity; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; -#ifdef ERTS_SMP - ASSERT(c_p->scheduler_data); -#endif - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); - nif_bif_result = (*fp)(&env, bif_nif_arity, reg); - if (env.exception_thrown) - nif_bif_result = THE_NON_VALUE; - erts_post_nif(&env); - ERTS_CHK_MBUF_SZ(c_p); - - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - ASSERT(!env.exiting); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } - - DTRACE_NIF_RETURN(c_p, codemfa); - goto apply_bif_or_nif_epilogue; - - OpCase(apply_bif): - /* - * At this point, I points to the code[0] in the export entry for - * the BIF: - * - * code[-3]: Module - * code[-2]: Function - * code[-1]: Arity - * code[0]: &&apply_bif - * code[1]: Function pointer to BIF function - */ - - if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the bif */ - goto context_switch; - } - - codemfa = erts_code_to_codemfa(I); - - ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - - - /* In case we apply process_info/1,2 or load_nif/1 */ - c_p->current = codemfa; - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ - DTRACE_BIF_ENTRY(c_p, codemfa); - - SWAPOUT; - ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); - c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = codemfa->arity; - ASSERT(bif_nif_arity <= 4); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - { - ErtsBifFunc bf = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - live_hf_end = c_p->mbuf; - ERTS_CHK_MBUF_SZ(c_p); - nif_bif_result = (*bf)(c_p, reg, I); - ERTS_CHK_MBUF_SZ(c_p); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || - is_non_value(nif_bif_result)); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - PROCESS_MAIN_CHK_LOCKS(c_p); - } - /* We have to update the cache if we are enabled in order - to make sure no book keeping is done after we disabled - msacc. We don't always do this as it is quite expensive. */ - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) - ERTS_MSACC_UPDATE_CACHE_X(); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, codemfa); - - apply_bif_or_nif_epilogue: - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - if (ERTS_IS_GC_DESIRED(c_p)) { - nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, - nif_bif_result, - reg, bif_nif_arity); - } - SWAPIN; /* There might have been a garbage collection. */ - FCALLS = c_p->fcalls; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - if (is_value(nif_bif_result)) { - r(0) = nif_bif_result; - CHECK_TERM(r(0)); - SET_I(c_p->cp); - c_p->cp = 0; - Goto(*I); - } else if (c_p->freason == TRAP) { - SET_I(c_p->i); - if (c_p->flags & F_HIBERNATE_SCHED) { - c_p->flags &= ~F_HIBERNATE_SCHED; - goto do_schedule; - } - Dispatch(); - } - I = handle_error(c_p, c_p->cp, reg, c_p->current); - goto post_error_handling; - } - } - - OpCase(i_get_sd): - { - Eterm arg; - Eterm result; - - GetArg1(0, arg); - result = erts_pd_hash_get(c_p, arg); - StoreBifResult(1, result); - } - - OpCase(i_get_hash_cId): - { - Eterm arg; - Eterm result; - - GetArg1(0, arg); - result = erts_pd_hash_get_with_hx(c_p, Arg(1), arg); - StoreBifResult(2, result); - } - - { - Eterm case_end_val; - - OpCase(case_end_x): - case_end_val = xb(Arg(0)); - c_p->fvalue = case_end_val; - c_p->freason = EXC_CASE_CLAUSE; - goto find_func_info; - } - - OpCase(if_end): - c_p->freason = EXC_IF_CLAUSE; - goto find_func_info; - OpCase(i_func_info_IaaI): { ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; @@ -3749,1367 +936,8 @@ do { \ goto handle_error; } - OpCase(try_case_end_s): - { - Eterm try_case_end_val; - GetArg1(0, try_case_end_val); - c_p->fvalue = try_case_end_val; - c_p->freason = EXC_TRY_CLAUSE; - goto find_func_info; - } - - /* - * Construction of binaries using new instructions. - */ - { - Eterm new_binary; - Eterm num_bits_term; - Uint num_bits; - Uint alloc; - Uint num_bytes; - - OpCase(i_bs_init_bits_heap_IIId): { - num_bits = Arg(0); - alloc = Arg(1); - I++; - goto do_bs_init_bits_known; - } - - OpCase(i_bs_init_bits_IId): { - num_bits = Arg(0); - alloc = 0; - goto do_bs_init_bits_known; - } - - OpCase(i_bs_init_bits_fail_heap_sIjId): { - GetArg1(0, num_bits_term); - alloc = Arg(1); - I += 2; - goto do_bs_init_bits; - } - - OpCase(i_bs_init_bits_fail_yjId): { - num_bits_term = yb(Arg(0)); - I++; - alloc = 0; - goto do_bs_init_bits; - } - OpCase(i_bs_init_bits_fail_xjId): { - num_bits_term = xb(Arg(0)); - I++; - alloc = 0; - /* FALL THROUGH */ - } - - /* num_bits_term = Term for number of bits to build (small/big) - * alloc = Number of words to allocate on heap - * Operands: Fail Live Dst - */ - - do_bs_init_bits: - if (is_small(num_bits_term)) { - Sint size = signed_val(num_bits_term); - if (size < 0) { - goto badarg; - } - num_bits = (Uint) size; - } else { - Uint bits; - - if (!term_to_Uint(num_bits_term, &bits)) { - c_p->freason = bits; - goto lb_Cl_error; - - } - num_bits = (Eterm) bits; - } - - /* num_bits = Number of bits to build - * alloc = Number of extra words to allocate on heap - * Operands: NotUsed Live Dst - */ - do_bs_init_bits_known: - num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; - if (num_bits & 7) { - alloc += ERL_SUB_BIN_SIZE; - } - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - alloc += heap_bin_size(num_bytes); - } else { - alloc += PROC_BIN_SIZE; - } - TestHeap(alloc, Arg(1)); - - /* num_bits = Number of bits to build - * num_bytes = Number of bytes to allocate in the binary - * alloc = Total number of words to allocate on heap - * Operands: NotUsed NotUsed Dst - */ - if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - hb = (ErlHeapBin *) HTOP; - HTOP += heap_bin_size(num_bytes); - hb->thing_word = header_heap_bin(num_bytes); - hb->size = num_bytes; - erts_current_bin = (byte *) hb->data; - new_binary = make_binary(hb); - - do_bits_sub_bin: - if (num_bits & 7) { - ErlSubBin* sb; - - sb = (ErlSubBin *) HTOP; - HTOP += ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = num_bytes - 1; - sb->bitsize = num_bits & 7; - sb->offs = 0; - sb->bitoffs = 0; - sb->is_writable = 0; - sb->orig = new_binary; - new_binary = make_binary(sb); - } - HEAP_SPACE_VERIFIED(0); - StoreBifResult(2, new_binary); - } else { - Binary* bptr; - ProcBin* pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(num_bytes); - erts_current_bin = (byte *) bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HTOP; - HTOP += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = num_bytes; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header*) pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); - new_binary = make_binary(pb); - goto do_bits_sub_bin; - } - } - - { - Eterm BsOp1, BsOp2; - - OpCase(i_bs_init_fail_heap_sIjId): { - GetArg1(0, BsOp1); - BsOp2 = Arg(1); - I += 2; - goto do_bs_init; - } - - OpCase(i_bs_init_fail_yjId): { - BsOp1 = yb(Arg(0)); - BsOp2 = 0; - I++; - goto do_bs_init; - } - - OpCase(i_bs_init_fail_xjId): { - BsOp1 = xb(Arg(0)); - BsOp2 = 0; - I++; - } - /* FALL THROUGH */ - do_bs_init: - if (is_small(BsOp1)) { - Sint size = signed_val(BsOp1); - if (size < 0) { - goto badarg; - } - BsOp1 = (Eterm) size; - } else { - Uint bytes; - - if (!term_to_Uint(BsOp1, &bytes)) { - c_p->freason = bytes; - goto lb_Cl_error; - } - if ((bytes >> (8*sizeof(Uint)-3)) != 0) { - goto system_limit; - } - BsOp1 = (Eterm) bytes; - } - if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { - goto do_heap_bin_alloc; - } else { - goto do_proc_bin_alloc; - } - - - OpCase(i_bs_init_heap_IIId): { - BsOp1 = Arg(0); - BsOp2 = Arg(1); - I++; - goto do_proc_bin_alloc; - } - - OpCase(i_bs_init_IId): { - BsOp1 = Arg(0); - BsOp2 = 0; - } - /* FALL THROUGH */ - do_proc_bin_alloc: { - Binary* bptr; - ProcBin* pb; - - erts_bin_offset = 0; - erts_writable_bin = 0; - TestBinVHeap(BsOp1 / sizeof(Eterm), - BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1)); - - /* - * Allocate the binary struct itself. - */ - bptr = erts_bin_nrml_alloc(BsOp1); - erts_current_bin = (byte *) bptr->orig_bytes; - - /* - * Now allocate the ProcBin on the heap. - */ - pb = (ProcBin *) HTOP; - HTOP += PROC_BIN_SIZE; - pb->thing_word = HEADER_PROC_BIN; - pb->size = BsOp1; - pb->next = MSO(c_p).first; - MSO(c_p).first = (struct erl_off_heap_header*) pb; - pb->val = bptr; - pb->bytes = (byte*) bptr->orig_bytes; - pb->flags = 0; - - OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); - - StoreBifResult(2, make_binary(pb)); - } - - OpCase(i_bs_init_heap_bin_heap_IIId): { - BsOp1 = Arg(0); - BsOp2 = Arg(1); - I++; - goto do_heap_bin_alloc; - } - - OpCase(i_bs_init_heap_bin_IId): { - BsOp1 = Arg(0); - BsOp2 = 0; - } - /* Fall through */ - do_heap_bin_alloc: - { - ErlHeapBin* hb; - Uint bin_need; - - bin_need = heap_bin_size(BsOp1); - erts_bin_offset = 0; - erts_writable_bin = 0; - TestHeap(bin_need+BsOp2+ERL_SUB_BIN_SIZE, Arg(1)); - hb = (ErlHeapBin *) HTOP; - HTOP += bin_need; - hb->thing_word = header_heap_bin(BsOp1); - hb->size = BsOp1; - erts_current_bin = (byte *) hb->data; - BsOp1 = make_binary(hb); - StoreBifResult(2, BsOp1); - } - } - - OpCase(bs_add_jssId): { - Eterm Op1, Op2; - Uint Unit = Arg(3); - - GetArg2(1, Op1, Op2); - if (is_both_small(Op1, Op2)) { - Sint Arg1 = signed_val(Op1); - Sint Arg2 = signed_val(Op2); - - if (Arg1 >= 0 && Arg2 >= 0) { - BsSafeMul(Arg2, Unit, goto system_limit, Op1); - Op1 += Arg1; - - store_bs_add_result: - if (Op1 <= MAX_SMALL) { - Op1 = make_small(Op1); - } else { - /* - * May generate a heap fragment, but in this - * particular case it is OK, since the value will be - * stored into an x register (the GC will scan x - * registers for references to heap fragments) and - * there is no risk that value can be stored into a - * location that is not scanned for heap-fragment - * references (such as the heap). - */ - SWAPOUT; - Op1 = erts_make_integer(Op1, c_p); - HTOP = HEAP_TOP(c_p); - } - StoreBifResult(4, Op1); - } - goto badarg; - } else { - Uint a; - Uint b; - Uint c; - - /* - * Now we know that one of the arguments is - * not a small. We must convert both arguments - * to Uints and check for errors at the same time. - * - * Error checking is tricky. - * - * If one of the arguments is not numeric or - * not positive, the error reason is BADARG. - * - * Otherwise if both arguments are numeric, - * but at least one argument does not fit in - * an Uint, the reason is SYSTEM_LIMIT. - */ - - if (!term_to_Uint(Op1, &a)) { - if (a == BADARG) { - goto badarg; - } - if (!term_to_Uint(Op2, &b)) { - c_p->freason = b; - goto lb_Cl_error; - } - goto system_limit; - } else if (!term_to_Uint(Op2, &b)) { - c_p->freason = b; - goto lb_Cl_error; - } - - /* - * The arguments are now correct and stored in a and b. - */ - - BsSafeMul(b, Unit, goto system_limit, c); - Op1 = a + c; - if (Op1 < a) { - /* - * If the result is less than one of the - * arguments, there must have been an overflow. - */ - goto system_limit; - } - goto store_bs_add_result; - } - /* No fallthrough */ - ASSERT(0); - } - - OpCase(bs_put_string_II): - { - BeamInstr *next; - PreFetch(2, next); - erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0))); - NextPF(2, next); - } - - /* - * x(SCRATCH_X_REG); - * Operands: Fail ExtraHeap Live Unit Size Dst - */ - - OpCase(i_bs_append_jIIIsd): { - Uint live = Arg(2); - Uint res; - Eterm Size; - - GetArg1(4, Size); - HEAVY_SWAPOUT; - reg[live] = x(SCRATCH_X_REG); - res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); - HEAVY_SWAPIN; - if (is_non_value(res)) { - /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ - goto lb_Cl_error; - } - StoreBifResult(5, res); - } - - /* - * Operands: Fail Size Src Unit Dst - */ - OpCase(i_bs_private_append_jIssd): { - Eterm res; - Eterm Size, Src; - - GetArg2(2, Size, Src); - res = erts_bs_private_append(c_p, Src, Size, Arg(1)); - if (is_non_value(res)) { - /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ - goto lb_Cl_error; - } - StoreBifResult(4, res); - } - - OpCase(bs_init_writable): { - HEAVY_SWAPOUT; - r(0) = erts_bs_init_writable(c_p, r(0)); - HEAVY_SWAPIN; - Next(0); - } - - /* - * Calculate the number of bytes needed to encode the source - * operarand to UTF-8. If the source operand is invalid (e.g. wrong - * type or range) we return a nonsense integer result (0 or 4). We - * can get away with that because we KNOW that bs_put_utf8 will do - * full error checking. - */ - OpCase(i_bs_utf8_size_sd): { - Eterm arg; - Eterm result; - - GetArg1(0, arg); - if (arg < make_small(0x80UL)) { - result = make_small(1); - } else if (arg < make_small(0x800UL)) { - result = make_small(2); - } else if (arg < make_small(0x10000UL)) { - result = make_small(3); - } else { - result = make_small(4); - } - StoreBifResult(1, result); - } - - OpCase(i_bs_put_utf8_js): { - Eterm arg; - - GetArg1(1, arg); - if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(arg))) { - goto badarg; - } - Next(2); - } - - /* - * Calculate the number of bytes needed to encode the source - * operarand to UTF-8. If the source operand is invalid (e.g. wrong - * type or range) we return a nonsense integer result (2 or 4). We - * can get away with that because we KNOW that bs_put_utf16 will do - * full error checking. - */ - - OpCase(i_bs_utf16_size_sd): { - Eterm arg; - Eterm result = make_small(2); - - GetArg1(0, arg); - if (arg >= make_small(0x10000UL)) { - result = make_small(4); - } - StoreBifResult(1, result); - } - - OpCase(bs_put_utf16_jIs): { - Eterm arg; - - GetArg1(2, arg); - if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(arg, Arg(1)))) { - goto badarg; - } - Next(3); - } - - /* - * Only used for validating a value about to be stored in a binary. - */ - OpCase(i_bs_validate_unicode_js): { - Eterm val; - - GetArg1(1, val); - - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (if the term is a bignum, it could - * slip through the test, and there is no further test that - * would catch it, since bit syntax construction silently masks - * too big numbers). - */ - if (is_not_small(val) || val > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) { - goto badarg; - } - Next(2); - } - - /* - * Only used for validating a value matched out. - */ - OpCase(i_bs_validate_unicode_retract_jss): { - Eterm i; /* Integer to validate */ - - /* - * There is no need to untag the integer, but it IS necessary - * to make sure it is small (a bignum pointer could fall in - * the valid range). - */ - - GetArg1(1, i); - if (is_not_small(i) || i > make_small(0x10FFFFUL) || - (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { - Eterm ms; /* Match context */ - ErlBinMatchBuffer* mb; - - GetArg1(2, ms); - mb = ms_matchbuffer(ms); - mb->offset -= 32; - goto badarg; - } - Next(3); - } - - /* - * Matching of binaries. - */ - - { - Eterm header; - BeamInstr *next; - Uint slots; - Eterm context; - - do_start_match: - slots = Arg(2); - if (!is_boxed(context)) { - ClauseFail(); - } - PreFetch(4, next); - header = *boxed_val(context); - if (header_is_bin_matchstate(header)) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); - Uint actual_slots = HEADER_NUM_SLOTS(header); - ms->save_offset[0] = ms->mb.offset; - if (actual_slots < slots) { - ErlBinMatchState* dst; - Uint live = Arg(1); - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - - TestHeapPreserve(wordsneeded, live, context); - ms = (ErlBinMatchState *) boxed_val(context); - dst = (ErlBinMatchState *) HTOP; - *dst = *ms; - *HTOP = HEADER_BIN_MATCHSTATE(slots); - HTOP += wordsneeded; - HEAP_SPACE_VERIFIED(0); - StoreResult(make_matchstate(dst), Arg(3)); - } - } else if (is_binary_header(header)) { - Eterm result; - Uint live = Arg(1); - Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, context); - HEAP_TOP(c_p) = HTOP; -#ifdef DEBUG - c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ -#endif - result = erts_bs_start_match_2(c_p, context, slots); - HTOP = HEAP_TOP(c_p); - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } else { - StoreResult(result, Arg(3)); - } - } else { - ClauseFail(); - } - NextPF(4, next); - - OpCase(i_bs_start_match2_xfIId): { - context = xb(Arg(0)); - I++; - goto do_start_match; - } - OpCase(i_bs_start_match2_yfIId): { - context = yb(Arg(0)); - I++; - goto do_start_match; - } - } - - OpCase(bs_test_zero_tail2_fx): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - - PreFetch(2, next); - _mb = (ErlBinMatchBuffer*) ms_matchbuffer(xb(Arg(1))); - if (_mb->size != _mb->offset) { - ClauseFail(); - } - NextPF(2, next); - } - - OpCase(bs_test_tail_imm2_fxI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(3, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if (_mb->size - _mb->offset != Arg(2)) { - ClauseFail(); - } - NextPF(3, next); - } - - OpCase(bs_test_unit_fxI): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(3, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if ((_mb->size - _mb->offset) % Arg(2)) { - ClauseFail(); - } - NextPF(3, next); - } - - OpCase(bs_test_unit8_fx): { - BeamInstr *next; - ErlBinMatchBuffer *_mb; - PreFetch(2, next); - _mb = ms_matchbuffer(xb(Arg(1))); - if ((_mb->size - _mb->offset) & 7) { - ClauseFail(); - } - NextPF(2, next); - } - - { - Eterm bs_get_integer8_context; - - OpCase(i_bs_get_integer_8_xfd): { - ErlBinMatchBuffer *_mb; - Eterm _result; - bs_get_integer8_context = xb(Arg(0)); - I++; - _mb = ms_matchbuffer(bs_get_integer8_context); - if (_mb->size - _mb->offset < 8) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); - } else { - _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); - _mb->offset += 8; - } - StoreBifResult(1, _result); - } - } - - { - Eterm bs_get_integer_16_context; - - OpCase(i_bs_get_integer_16_xfd): - bs_get_integer_16_context = xb(Arg(0)); - I++; - - { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(bs_get_integer_16_context); - if (_mb->size - _mb->offset < 16) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); - } else { - _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); - _mb->offset += 16; - } - StoreBifResult(1, _result); - } - } - - { - Eterm bs_get_integer_32_context; - - OpCase(i_bs_get_integer_32_xfId): - bs_get_integer_32_context = xb(Arg(0)); - I++; - - { - ErlBinMatchBuffer *_mb; - Uint32 _integer; - Eterm _result; - _mb = ms_matchbuffer(bs_get_integer_32_context); - if (_mb->size - _mb->offset < 32) { ClauseFail(); } - if (BIT_OFFSET(_mb->offset) != 0) { - _integer = erts_bs_get_unaligned_uint32(_mb); - } else { - _integer = get_int32(_mb->base + _mb->offset/8); - } - _mb->offset += 32; -#if !defined(ARCH_64) - if (IS_USMALL(0, _integer)) { -#endif - _result = make_small(_integer); -#if !defined(ARCH_64) - } else { - TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); - _result = uint_to_big((Uint) _integer, HTOP); - HTOP += BIG_UINT_HEAP_SIZE; - HEAP_SPACE_VERIFIED(0); - } -#endif - StoreBifResult(2, _result); - } - } - - { - Eterm Ms, Sz; - - /* Operands: x(Reg) Size Live Fail Flags Dst */ - OpCase(i_bs_get_integer_imm_xIIfId): { - Uint wordsneeded; - Ms = xb(Arg(0)); - Sz = Arg(1); - wordsneeded = 1+WSIZE(NBYTES(Sz)); - TestHeapPreserve(wordsneeded, Arg(2), Ms); - I += 3; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* Operands: x(Reg) Size Fail Flags Dst */ - OpCase(i_bs_get_integer_small_imm_xIfId): { - Ms = xb(Arg(0)); - Sz = Arg(1); - I += 2; - /* Operands: Fail Flags Dst */ - goto do_bs_get_integer_imm; - } - - /* - * Ms = match context - * Sz = size of field - * Operands: Fail Flags Dst - */ - do_bs_get_integer_imm: { - ErlBinMatchBuffer* mb; - Eterm result; - - mb = ms_matchbuffer(Ms); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, Sz, Arg(1), mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } - StoreBifResult(2, result); - } - } - - /* - * Operands: Fail Live FlagsAndUnit Ms Sz Dst - */ - OpCase(i_bs_get_integer_fIIssd): { - Uint flags; - Uint size; - Eterm Ms; - Eterm Sz; - ErlBinMatchBuffer* mb; - Eterm result; - - flags = Arg(2); - GetArg2(3, Ms, Sz); - BsGetFieldSize(Sz, (flags >> 3), ClauseFail(), size); - if (size >= SMALL_BITS) { - Uint wordsneeded; - /* Check bits size before potential gc. - * We do not want a gc and then realize we don't need - * the allocated space (i.e. if the op fails). - * - * Remember to re-acquire the matchbuffer after gc. - */ - - mb = ms_matchbuffer(Ms); - if (mb->size - mb->offset < size) { - ClauseFail(); - } - wordsneeded = 1+WSIZE(NBYTES((Uint) size)); - TestHeapPreserve(wordsneeded, Arg(1), Ms); - } - mb = ms_matchbuffer(Ms); - LIGHT_SWAPOUT; - result = erts_bs_get_integer_2(c_p, size, flags, mb); - LIGHT_SWAPIN; - HEAP_SPACE_VERIFIED(0); - if (is_non_value(result)) { - ClauseFail(); - } - StoreBifResult(5, result); - } - - { - Eterm get_utf8_context; - - /* Operands: MatchContext Fail Dst */ - OpCase(i_bs_get_utf8_xfd): { - get_utf8_context = xb(Arg(0)); - I++; - } - - /* - * get_utf8_context = match_context - * Operands: Fail Dst - */ - - { - Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); - if (is_non_value(result)) { - ClauseFail(); - } - StoreBifResult(1, result); - } - } - - { - Eterm get_utf16_context; - - /* Operands: MatchContext Fail Flags Dst */ - OpCase(i_bs_get_utf16_xfId): { - get_utf16_context = xb(Arg(0)); - I++; - } - - /* - * get_utf16_context = match_context - * Operands: Fail Flags Dst - */ - { - Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), - Arg(1)); - if (is_non_value(result)) { - ClauseFail(); - } - StoreBifResult(2, result); - } - } - - { - Eterm context_to_binary_context; - ErlBinMatchBuffer* mb; - ErlSubBin* sb; - Uint size; - Uint offs; - Uint orig; - Uint hole_size; - - OpCase(bs_context_to_binary_x): - context_to_binary_context = xb(Arg(0)); - I--; - - if (is_boxed(context_to_binary_context) && - header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { - ErlBinMatchState* ms; - ms = (ErlBinMatchState *) boxed_val(context_to_binary_context); - mb = &ms->mb; - offs = ms->save_offset[0]; - size = mb->size - offs; - goto do_bs_get_binary_all_reuse_common; - } - Next(2); - - OpCase(i_bs_get_binary_all_reuse_xfI): { - context_to_binary_context = xb(Arg(0)); - I++; - } - - mb = ms_matchbuffer(context_to_binary_context); - size = mb->size - mb->offset; - if (size % Arg(1) != 0) { - ClauseFail(); - } - offs = mb->offset; - - do_bs_get_binary_all_reuse_common: - orig = mb->orig; - sb = (ErlSubBin *) boxed_val(context_to_binary_context); - hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; - sb->thing_word = HEADER_SUB_BIN; - sb->size = BYTE_OFFSET(size); - sb->bitsize = BIT_OFFSET(size); - sb->offs = BYTE_OFFSET(offs); - sb->bitoffs = BIT_OFFSET(offs); - sb->is_writable = 0; - sb->orig = orig; - if (hole_size) { - sb[1].thing_word = make_pos_bignum_header(hole_size-1); - } - Next(2); - } - - { - Eterm match_string_context; - - OpCase(i_bs_match_string_xfII): { - match_string_context = xb(Arg(0)); - I++; - } - - { - BeamInstr *next; - byte* bytes; - Uint bits; - ErlBinMatchBuffer* mb; - Uint offs; - - PreFetch(3, next); - bits = Arg(1); - bytes = (byte *) Arg(2); - mb = ms_matchbuffer(match_string_context); - if (mb->size - mb->offset < bits) { - ClauseFail(); - } - offs = mb->offset & 7; - if (offs == 0 && (bits & 7) == 0) { - if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) { - ClauseFail(); - } - } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) { - ClauseFail(); - } - mb->offset += bits; - NextPF(3, next); - } - } - - OpCase(i_bs_save2_xI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(2, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); - _ms->save_offset[Arg(1)] = _ms->mb.offset; - NextPF(2, next); - } - - OpCase(i_bs_restore2_xI): { - BeamInstr *next; - ErlBinMatchState *_ms; - PreFetch(2, next); - _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0))); - _ms->mb.offset = _ms->save_offset[Arg(1)]; - NextPF(2, next); - } - #include "beam_cold.h" - - /* - * This instruction is probably never used (because it is combined with a - * a return). However, a future compiler might for some reason emit a - * deallocate not followed by a return, and that should work. - */ - OpCase(deallocate_I): { - BeamInstr *next; - - PreFetch(1, next); - D(Arg(0)); - NextPF(1, next); - } - - /* - * Trace and debugging support. - */ - - OpCase(return_trace): { - ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); - - SWAPOUT; /* Needed for shared heap */ - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[2])); - E += 3; - Goto(*I); - } - - OpCase(i_generic_breakpoint): { - BeamInstr real_I; - HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); - HEAVY_SWAPIN; - ASSERT(VALID_INSTR(real_I)); - Goto(real_I); - } - - OpCase(i_return_time_trace): { - BeamInstr *pc = (BeamInstr *) (UWord) E[0]; - SWAPOUT; - erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); - SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[1])); - E += 2; - Goto(*I); - } - - OpCase(i_return_to_trace): { - if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { - Uint *cpp = (Uint*) E; - for(;;) { - ASSERT(is_CP(*cpp)); - if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) { - do ++cpp; while(is_not_CP(*cpp)); - cpp += 2; - } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { - do ++cpp; while(is_not_CP(*cpp)); - } else break; - } - SWAPOUT; /* Needed for shared heap */ - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return_to(c_p, cp_val(*cpp)); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - SWAPIN; - } - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[0])); - E += 1; - Goto(*I); - } - - /* - * New floating point instructions. - */ - - OpCase(fmove_ql): { - Eterm fr = Arg(1); - BeamInstr *next; - - PreFetch(2, next); - GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - NextPF(2, next); - } - - OpCase(fmove_dl): { - Eterm targ1; - Eterm fr = Arg(1); - BeamInstr *next; - - PreFetch(2, next); - targ1 = REG_TARGET(Arg(0)); - /* Arg(0) == HEADER_FLONUM */ - GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - NextPF(2, next); - } - - OpCase(fmove_ld): { - Eterm fr = Arg(0); - Eterm dest = make_float(HTOP); - - PUT_DOUBLE(*(FloatDef*)ADD_BYTE_OFFSET(freg, fr), HTOP); - HTOP += FLOAT_SIZE_OBJECT; - StoreBifResult(1, dest); - } - - OpCase(fconv_dl): { - Eterm targ1; - Eterm fr = Arg(1); - BeamInstr *next; - - targ1 = REG_TARGET(Arg(0)); - PreFetch(2, next); - if (is_small(targ1)) { - fb(fr) = (double) signed_val(targ1); - } else if (is_big(targ1)) { - if (big_to_double(targ1, &fb(fr)) < 0) { - goto fbadarith; - } - } else if (is_float(targ1)) { - GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr)); - } else { - goto fbadarith; - } - NextPF(2, next); - } - -#ifdef NO_FPE_SIGNALS - OpCase(fclearerror): - OpCase(i_fcheckerror): - erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); -# define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT -# define ERTS_NO_FPE_ERROR ERTS_FP_ERROR -#else -# define ERTS_NO_FPE_CHECK_INIT(p) -# define ERTS_NO_FPE_ERROR(p, a, b) - - OpCase(fclearerror): { - BeamInstr *next; - - PreFetch(0, next); - ERTS_FP_CHECK_INIT(c_p); - NextPF(0, next); - } - - OpCase(i_fcheckerror): { - BeamInstr *next; - - PreFetch(0, next); - ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith); - NextPF(0, next); - } -#endif - - - OpCase(i_fadd_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fsub_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fmul_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fdiv_lll): { - BeamInstr *next; - - PreFetch(3, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1)); - ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith); - NextPF(3, next); - } - OpCase(i_fnegate_ll): { - BeamInstr *next; - - PreFetch(2, next); - ERTS_NO_FPE_CHECK_INIT(c_p); - fb(Arg(1)) = -fb(Arg(0)); - ERTS_NO_FPE_ERROR(c_p, fb(Arg(1)), goto fbadarith); - NextPF(2, next); - - fbadarith: - c_p->freason = BADARITH; - goto find_func_info; - } - -#ifdef HIPE - { -#define HIPE_MODE_SWITCH(Cmd) \ - SWAPOUT; \ - ERTS_DBG_CHK_REDS(c_p, FCALLS); \ - c_p->fcalls = FCALLS; \ - c_p->def_arg_reg[4] = -neg_o_reds; \ - c_p = hipe_mode_switch(c_p, Cmd, reg); \ - goto L_post_hipe_mode_switch - - OpCase(hipe_trap_call): { - /* - * I[-5]: &&lb_i_func_info_IaaI - * I[-4]: Native code callee (inserted by HiPE) - * I[-3]: Module (tagged atom) - * I[-2]: Function (tagged atom) - * I[-1]: Arity (untagged integer) - * I[ 0]: &&lb_hipe_trap_call - * ... remainder of original BEAM code - */ - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = ci->u.ncallee; - ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); - } - OpCase(hipe_trap_call_closure): { - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = ci->u.ncallee; - ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); - } - OpCase(hipe_trap_return): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); - } - OpCase(hipe_trap_throw): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); - } - OpCase(hipe_trap_resume): { - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); - } -#undef HIPE_MODE_SWITCH - - L_post_hipe_mode_switch: -#ifdef DEBUG - pid = c_p->common.id; /* may have switched process... */ -#endif - reg = erts_proc_sched_data(c_p)->x_reg_array; - freg = erts_proc_sched_data(c_p)->f_reg_array; - ERL_BITS_RELOAD_STATEP(c_p); - /* XXX: this abuse of def_arg_reg[] is horrid! */ - neg_o_reds = -c_p->def_arg_reg[4]; - FCALLS = c_p->fcalls; - SWAPIN; - ERTS_DBG_CHK_REDS(c_p, FCALLS); - switch( c_p->def_arg_reg[3] ) { - case HIPE_MODE_SWITCH_RES_RETURN: - ASSERT(is_value(reg[0])); - SET_I(c_p->cp); - c_p->cp = 0; - Goto(*I); - case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: - c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; - /*fall through*/ - case HIPE_MODE_SWITCH_RES_CALL_BEAM: - SET_I(c_p->i); - Dispatch(); - case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: - /* This can be used to call any function value, but currently it's - only used to call closures referring to unloaded modules. */ - { - BeamInstr *next; - - next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - HEAVY_SWAPIN; - if (next != NULL) { - SET_I(next); - Dispatchfun(); - } - goto find_func_info; - } - case HIPE_MODE_SWITCH_RES_THROW: - c_p->cp = NULL; - I = handle_error(c_p, I, reg, NULL); - goto post_error_handling; - default: - erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); - } - } - OpCase(hipe_call_count): { - /* - * I[-5]: &&lb_i_func_info_IaaI - * I[-4]: pointer to struct hipe_call_count (inserted by HiPE) - * I[-3]: Module (tagged atom) - * I[-2]: Function (tagged atom) - * I[-1]: Arity (untagged integer) - * I[ 0]: &&lb_hipe_call_count - * ... remainder of original BEAM code - */ - ErtsCodeInfo *ci = erts_code_to_codeinfo(I); - struct hipe_call_count *hcc = ci->u.hcc; - ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); - ASSERT(hcc != NULL); - ASSERT(VALID_INSTR(hcc->opcode)); - ++(hcc->count); - Goto(hcc->opcode); - } -#endif /* HIPE */ - - OpCase(i_yield): - { - /* This is safe as long as REDS_IN(c_p) is never stored - * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5], - * which may be c_p->arg_reg[5], which is close, but no banana. - */ - c_p->arg_reg[0] = am_true; - c_p->arity = 1; /* One living register (the 'true' return value) */ - SWAPOUT; - c_p->i = I + 1; /* Next instruction */ - c_p->current = NULL; - goto do_schedule; - } - - OpCase(i_hibernate): { - HEAVY_SWAPOUT; - if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { - FCALLS = c_p->fcalls; - c_p->flags &= ~F_HIBERNATE_SCHED; - goto do_schedule; - } else { - HEAVY_SWAPIN; - I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); - goto post_error_handling; - } - } - - /* This is optimised as an instruction because - it has to be very very fast */ - OpCase(i_perf_counter): { - BeamInstr* next; - ErtsSysPerfCounter ts; - PreFetch(0, next); - - ts = erts_sys_perf_counter(); - - if (IS_SSMALL(ts)) { - r(0) = make_small((Sint)ts); - } else { - TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); - r(0) = make_big(HTOP); -#if defined(ARCH_32) - if (ts >= (((Uint64) 1) << 32)) { - *HTOP = make_pos_bignum_header(2); - BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); - BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); - HTOP += 3; - } - else -#endif - { - *HTOP = make_pos_bignum_header(1); - BIG_DIGIT(HTOP, 0) = (Uint) ts; - HTOP += 2; - } - } - NextPF(0, next); - } - - OpCase(i_debug_breakpoint): { - HEAVY_SWAPOUT; - I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); - HEAVY_SWAPIN; - if (I) { - Goto(*I); - } - goto handle_error; - } - - - OpCase(system_limit_j): - system_limit: - c_p->freason = SYSTEM_LIMIT; - goto lb_Cl_error; - - #ifdef ERTS_OPCODE_COUNTER_SUPPORT DEFINE_COUNTING_LABELS; #endif @@ -5132,9 +960,6 @@ do { \ init_emulator: { - int i; - Export* ep; - #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT #ifdef DEBUG @@ -5146,35 +971,8 @@ do { \ beam_ops = opcodes; #endif /* ERTS_OPCODE_COUNTER_SUPPORT */ #endif /* NO_JUMP_TABLE */ - - em_call_error_handler = OpCode(call_error_handler); - em_apply_bif = OpCode(apply_bif); - em_call_nif = OpCode(call_nif); - em_call_bif_e = OpCode(call_bif_e); - - beam_apply[0] = (BeamInstr) OpCode(i_apply); - beam_apply[1] = (BeamInstr) OpCode(normal_exit); - beam_exit[0] = (BeamInstr) OpCode(error_action_code); - beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); - beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); - beam_return_trace[0] = (BeamInstr) OpCode(return_trace); - beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ - beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace); - - /* - * Enter all BIFs into the export table. - */ - for (i = 0; i < BIF_SIZE; i++) { - ep = erts_export_put(bif_table[i].module, - bif_table[i].name, - bif_table[i].arity); - bif_export[i] = ep; - ep->beam[0] = (BeamInstr) OpCode(apply_bif); - ep->beam[1] = (BeamInstr) bif_table[i].f; - /* XXX: set func info for bifs */ - ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); - } + init_emulator_finish(); return; } #ifdef NO_JUMP_TABLE @@ -5186,26 +984,59 @@ do { \ save_calls1: { - Eterm* dis_next; + BeamInstr dis_next; save_calls(c_p, (Export *) Arg(0)); SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); - dis_next = (Eterm *) *I; + dis_next = *I; FCALLS--; Goto(dis_next); } } /* + * One-time initialization of emulator. Does not need to be + * in process_main(). + */ +static void +init_emulator_finish(void) +{ + int i; + Export* ep; + + beam_apply[0] = BeamOpCodeAddr(op_i_apply); + beam_apply[1] = BeamOpCodeAddr(op_normal_exit); + beam_exit[0] = BeamOpCodeAddr(op_error_action_code); + beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); + beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); + beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); + beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ + beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); + + /* + * Enter all BIFs into the export table. + */ + for (i = 0; i < BIF_SIZE; i++) { + ep = erts_export_put(bif_table[i].module, + bif_table[i].name, + bif_table[i].arity); + bif_export[i] = ep; + ep->beam[0] = BeamOpCodeAddr(op_apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; + /* XXX: set func info for bifs */ + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); + } +} + +/* * erts_dirty_process_main() is what dirty schedulers execute. Since they handle * only NIF calls they do not need to be able to execute all BEAM * instructions. */ void erts_dirty_process_main(ErtsSchedulerData *esdp) { -#ifdef ERTS_DIRTY_SCHEDULERS Process* c_p = NULL; ErtsMonotonicTime start_time; #ifdef DEBUG @@ -5317,7 +1148,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) } PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = erts_schedule(esdp, c_p, reds_used); @@ -5331,7 +1162,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) #ifdef DEBUG pid = c_p->common.id; /* Save for debugging purposes */ #endif - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!(c_p->flags & F_HIPE_MODE)); @@ -5346,7 +1177,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) else c_p->fcalls = CONTEXT_REDS; - if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) { + if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) { erts_execute_dirty_system_task(c_p); goto do_dirty_schedule; } @@ -5417,21 +1248,21 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - if (em_apply_bif == (BeamInstr *) *I) { + if (BeamIsOpCode(*I, op_apply_bif)) { exiting = erts_call_dirty_bif(esdp, c_p, I, reg); } else { - ASSERT(em_call_nif == (BeamInstr *) *I); - exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + ASSERT(BeamIsOpCode(*I, op_call_nif)); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); } ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); if (exiting) @@ -5444,7 +1275,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; goto context_switch; } -#endif /* ERTS_DIRTY_SCHEDULERS */ } static ErtsCodeMFA * @@ -5610,9 +1440,9 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) } if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); return NULL; } @@ -6261,8 +2091,8 @@ apply_bif_error_adjustment(Process *p, Export *ep, * and apply_last_IP. */ if (I - && ep->beam[0] == (BeamInstr) em_apply_bif - && (ep == bif_export[BIF_error_1] + && BeamIsOpCode(ep->beam[0], op_apply_bif) + && (ep == bif_export[BIF_error_1] || ep == bif_export[BIF_error_2] || ep == bif_export[BIF_exit_1] || ep == bif_export[BIF_throw_1])) { @@ -6348,13 +2178,14 @@ apply_bif_error_adjustment(Process *p, Export *ep, } static BeamInstr* -apply( -Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg, -BeamInstr *I, Uint stack_offset) +apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset) { int arity; Export* ep; - Eterm tmp, this; + Eterm tmp; + Eterm module = reg[0]; + Eterm function = reg[1]; + Eterm args = reg[2]; /* * Check the arguments which should be of the form apply(Module, @@ -6377,20 +2208,8 @@ BeamInstr *I, Uint stack_offset) while (1) { Eterm m, f, a; - /* The module argument may be either an atom or an abstract module - * (currently implemented using tuples, but this might change). - */ - this = THE_NON_VALUE; - if (is_not_atom(module)) { - Eterm* tp; - - if (is_not_tuple(module)) goto error; - tp = tuple_val(module); - if (arityval(tp[0]) < 1) goto error; - this = module; - module = tp[1]; - if (is_not_atom(module)) goto error; - } + + if (is_not_atom(module)) goto error; if (module != am_erlang || function != am_apply) break; @@ -6425,9 +2244,7 @@ BeamInstr *I, Uint stack_offset) } /* * Walk down the 3rd parameter of apply (the argument list) and copy - * the parameters to the x registers (reg[]). If the module argument - * was an abstract module, add 1 to the function arity and put the - * module argument in the n+1st x register as a THIS reference. + * the parameters to the x registers (reg[]). */ tmp = args; @@ -6444,9 +2261,6 @@ BeamInstr *I, Uint stack_offset) if (is_not_nil(tmp)) { /* Must be well-formed list */ goto error; } - if (this != THE_NON_VALUE) { - reg[arity++] = this; - } /* * Get the index into the export table, or failing that the export @@ -6485,22 +2299,12 @@ fixed_apply(Process* p, Eterm* reg, Uint arity, return 0; } - /* The module argument may be either an atom or an abstract module - * (currently implemented using tuples, but this might change). - */ - if (is_not_atom(module)) { - Eterm* tp; - if (is_not_tuple(module)) goto error; - tp = tuple_val(module); - if (arityval(tp[0]) < 1) goto error; - module = tp[1]; - if (is_not_atom(module)) goto error; - ++arity; - } + if (is_not_atom(module)) goto error; /* Handle apply of apply/3... */ - if (module == am_erlang && function == am_apply && arity == 3) - return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset); + if (module == am_erlang && function == am_apply && arity == 3) { + return apply(p, reg, I, stack_offset); + } /* * Get the index into the export table, or failing that the export @@ -6521,27 +2325,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity, } int -erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) +erts_hibernate(Process* c_p, Eterm* reg) { int arity; Eterm tmp; - -#ifndef ERTS_SMP - if (ERTS_PROC_IS_EXITING(c_p)) { - /* - * I non smp case: - * - * Currently executing process might be sent an exit - * signal if it is traced by a port that it also is - * linked to, and the port terminates during the - * trace. In this case we do *not* want to clear - * the active flag, which will make the process hang - * in limbo forever. Get out of here and terminate - * the process... - */ - return -1; - } -#endif + Eterm module = reg[0]; + Eterm function = reg[1]; + Eterm args = reg[2]; if (is_not_atom(module) || is_not_atom(function)) { /* @@ -6609,33 +2399,22 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re * If there are no waiting messages, garbage collect and * shrink the heap. */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); erts_garbage_collect_hibernate(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); -#ifndef ERTS_SMP - if (ERTS_PROC_IS_EXITING(c_p)) { - /* - * See comment in the beginning of the function... - * - * This second test is needed since gc might be traced. - */ - return -1; - } -#else /* ERTS_SMP */ - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); if (!c_p->msg.len) -#endif - erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); + erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; @@ -6725,7 +2504,7 @@ call_fun(Process* p, /* Current process. */ module = fe->module; - ERTS_SMP_READ_MEMORY_BARRIER; + ERTS_THR_READ_MEMORY_BARRIER; if (fe->pend_purge_address) { /* * The system is currently trying to purge the @@ -6829,7 +2608,7 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) } if (is_not_nil(tmp)) { /* Must be well-formed list */ - p->freason = EXC_UNDEF; + p->freason = EXC_BADARG; return NULL; } reg[arity] = fun; @@ -6856,7 +2635,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) p->htop = hp + needed; funp = (ErlFunThing *) hp; hp = funp->env; - erts_smp_refc_inc(&fe->refc, 2); + erts_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; funp->next = MSO(p).first; MSO(p).first = (struct erl_off_heap_header*) funp; @@ -6959,24 +2738,20 @@ do { \ static Eterm -new_map(Process* p, Eterm* reg, BeamInstr* I) +new_map(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* ptr) { - Uint n = Arg(3); Uint i; Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; Eterm keys; Eterm *mhp,*thp; Eterm *E; - BeamInstr *ptr; flatmap_t *mp; ErtsHeapFactory factory; - ptr = &Arg(4); - if (n > 2*MAP_SMALL_MAP_LIMIT) { Eterm res; if (HeapWordsLeft(p) < n) { - erts_garbage_collect(p, n, reg, Arg(2)); + erts_garbage_collect(p, n, reg, live); } mhp = p->htop; @@ -6997,7 +2772,7 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) } if (HeapWordsLeft(p) < need) { - erts_garbage_collect(p, need, reg, Arg(2)); + erts_garbage_collect(p, need, reg, live); } thp = p->htop; @@ -7020,9 +2795,42 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) } static Eterm -update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +new_small_map_lit(Process* p, Eterm* reg, Eterm keys_literal, Uint live, BeamInstr* ptr) +{ + Eterm* keys = tuple_val(keys_literal); + Uint n = arityval(*keys); + Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */; + Uint i; + flatmap_t *mp; + Eterm *mhp; + Eterm *E; + + ASSERT(n <= MAP_SMALL_MAP_LIMIT); + + if (HeapWordsLeft(p) < need) { + erts_garbage_collect(p, need, reg, live); + } + + mhp = p->htop; + E = p->stop; + + mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys_literal; + + for (i = 0; i < n; i++) { + GET_TERM(*ptr++, *mhp++); + } + + p->htop = mhp; + + return make_flatmap(mp); +} + +static Eterm +update_map_assoc(Process* p, Eterm* reg, Uint live, Uint n, BeamInstr* new_p) { - Uint n; Uint num_old; Uint num_updates; Uint need; @@ -7032,23 +2840,18 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm* E; Eterm* old_keys; Eterm* old_vals; - BeamInstr* new_p; Eterm new_key; Eterm* kp; + Eterm map; - new_p = &Arg(5); - num_updates = Arg(4) / 2; + num_updates = n / 2; + map = reg[live]; if (is_not_flatmap(map)) { Uint32 hx; Eterm val; - /* apparently the compiler does not emit is_map instructions, - * bad compiler */ - - if (is_not_hashmap(map)) - return THE_NON_VALUE; - + ASSERT(is_hashmap(map)); res = map; E = p->stop; while(num_updates--) { @@ -7072,7 +2875,7 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) */ if (num_old == 0) { - return new_map(p, reg, I+1); + return new_map(p, reg, live, n, new_p); } /* @@ -7082,8 +2885,6 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { - Uint live = Arg(3); - reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; old_mp = (flatmap_t *)flatmap_val(map); @@ -7230,9 +3031,8 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) */ static Eterm -update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) +update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p) { - Uint n; Uint i; Uint num_old; Uint need; @@ -7242,12 +3042,12 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) Eterm* E; Eterm* old_keys; Eterm* old_vals; - BeamInstr* new_p; Eterm new_key; + Eterm map; - new_p = &Arg(5); - n = Arg(4) / 2; /* Number of values to be updated */ + n /= 2; /* Number of values to be updated */ ASSERT(n > 0); + map = reg[live]; if (is_not_flatmap(map)) { Uint32 hx; @@ -7301,8 +3101,6 @@ update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I) need = num_old + MAP_HEADER_FLATMAP_SZ; if (HeapWordsLeft(p) < need) { - Uint live = Arg(3); - reg[live] = map; erts_garbage_collect(p, need, reg, live+1); map = reg[live]; old_mp = (flatmap_t *)flatmap_val(map); @@ -7400,8 +3198,8 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->beam - && (ep->beam[0] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam && + BeamIsOpCode(ep->beam[0], op_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 23258dbe9c..9835b1c096 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -55,12 +55,6 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define DEFINED 1 #define EXPORTED 2 -#ifdef NO_JUMP_TABLE -# define BeamOpCode(Op) ((BeamInstr)(Op)) -#else -# define BeamOpCode(Op) ((BeamInstr)beam_ops[Op]) -#endif - #if defined(WORDS_BIGENDIAN) # define NATIVE_ENDIAN(F) \ if ((F).val & BSF_NATIVE) { \ @@ -81,16 +75,28 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int); #define TE_FAIL (-1) #define TE_SHORT_WINDOW (-2) +/* + * Type for a reference to a label that must be patched. + */ + typedef struct { - Uint value; /* Value of label (NULL if not known yet). */ - Sint patches; /* Index (into code buffer) to first location - * which must be patched with the value of this label. - */ -#ifdef ERTS_SMP + Uint pos; /* Position of label reference to patch. */ + Uint offset; /* Offset from patch location. */ + int packed; /* 0 (not packed), 1 (lsw), 2 (msw) */ +} LabelPatch; + +/* + * Type for a label. + */ + +typedef struct { + Uint value; /* Value of label (0 if not known yet). */ Uint looprec_targeted; /* Non-zero if this label is the target of a loop_rec * instruction. */ -#endif + LabelPatch* patches; /* Array of label patches. */ + Uint num_patches; /* Number of patches in array. */ + Uint num_allocated; /* Number of allocated patches. */ } Label; /* @@ -227,7 +233,7 @@ typedef struct { typedef struct literal_patch LiteralPatch; struct literal_patch { - int pos; /* Position in code */ + Uint pos; /* Position in code */ LiteralPatch* next; }; @@ -307,6 +313,7 @@ typedef struct LoaderState { int on_load; /* Index in the code for the on_load function * (or 0 if there is no on_load function) */ + int otp_20_or_higher; /* Compiled with OTP 20 or higher */ /* * Atom table. @@ -507,6 +514,7 @@ static int read_lambda_table(LoaderState* stp); static int read_literal_table(LoaderState* stp); static int read_line_table(LoaderState* stp); static int read_code_header(LoaderState* stp); +static void init_label(Label* lp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOpArg Tuple, GenOpArg Dst); @@ -537,6 +545,7 @@ static int get_tag_and_value(LoaderState* stp, Uint len_code, static int new_label(LoaderState* stp); static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); +static int find_literal(LoaderState* stp, Eterm needle, Uint *idx); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, @@ -740,6 +749,13 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, } /* + * Find out whether the code was compiled with OTP 20 + * or higher. + */ + + stp->otp_20_or_higher = stp->chunks[UTF8_ATOM_CHUNK].size > 0; + + /* * Load the code chunk. */ @@ -795,8 +811,8 @@ erts_finish_loading(Binary* magic, Process* c_p, * table which is not protected by any locks. */ - ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() || - erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission() || + erts_thr_progress_is_blocking()); /* * Make current code for the module old and insert the new code * as current. This will fail if there already exists old code @@ -827,11 +843,10 @@ erts_finish_loading(Binary* magic, Process* c_p, continue; } if (ep->addressv[code_ix] == ep->beam) { - if (ep->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { continue; - } else if (ep->beam[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + } else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); erts_clear_export_break(mod_tab_p, &ep->info); ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; @@ -1043,6 +1058,10 @@ loader_state_dtor(Binary* magic) stp->codev = 0; } if (stp->labels != 0) { + Uint num; + for (num = 0; num < stp->num_labels; num++) { + erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels[num].patches); + } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels); stp->labels = 0; } @@ -1456,7 +1475,7 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->beam[0] == (BeamInstr) em_apply_bif) { + if (BeamIsOpCode(e->beam[0], op_apply_bif)) { stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; @@ -1526,7 +1545,7 @@ read_export_table(LoaderState* stp) * any other functions that walk through all local functions. */ - if (stp->labels[n].patches >= 0) { + if (stp->labels[n].num_patches > 0) { LoadError3(stp, "there are local calls to the stub for " "the BIF %T:%T/%d", stp->module, func, arity); @@ -1550,7 +1569,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->beam[0] != (BeamInstr) em_apply_bif) { + if (! BeamIsOpCode(e->beam[0], op_apply_bif)) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1872,11 +1891,7 @@ read_code_header(LoaderState* stp) stp->labels = (Label *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_labels * sizeof(Label)); for (i = 0; i < stp->num_labels; i++) { - stp->labels[i].value = 0; - stp->labels[i].patches = -1; -#ifdef ERTS_SMP - stp->labels[i].looprec_targeted = 0; -#endif + init_label(&stp->labels[i]); } stp->catches = 0; @@ -1905,12 +1920,43 @@ read_code_header(LoaderState* stp) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) +static void init_label(Label* lp) +{ + lp->value = 0; + lp->looprec_targeted = 0; + lp->num_patches = 0; + lp->num_allocated = 4; + lp->patches = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + lp->num_allocated * sizeof(LabelPatch)); +} + +static void +register_label_patch(LoaderState* stp, Uint label, Uint ci, Uint offset) +{ + Label* lp; + + ASSERT(label < stp->num_labels); + lp = &stp->labels[label]; + if (lp->num_allocated <= lp->num_patches) { + lp->num_allocated *= 2; + lp->patches = erts_realloc(ERTS_ALC_T_PREPARED_CODE, + (void *) lp->patches, + lp->num_allocated * sizeof(LabelPatch)); + } + lp->patches[lp->num_patches].pos = ci; + lp->patches[lp->num_patches].offset = offset; + lp->patches[lp->num_patches].packed = 0; + lp->num_patches++; + stp->codev[ci] = label; +} + static int load_code(LoaderState* stp) { int i; - int ci; - int last_func_start = 0; /* Needed by nif loading and line instructions */ + Uint ci; + Uint last_instr_start; /* Needed for relative jumps */ + Uint last_func_start = 0; /* Needed by nif loading and line instructions */ char* sign; int arg; /* Number of current argument. */ int num_specific; /* Number of specific ops for current. */ @@ -1923,6 +1969,9 @@ load_code(LoaderState* stp) GenOp** last_op_next = NULL; int arity; int retval = 1; +#if defined(BEAM_WIDE_SHIFT) + int num_trailing_f; /* Number of extra 'f' arguments in a list */ +#endif /* * The size of the loaded func_info instruction is needed @@ -2028,30 +2077,10 @@ load_code(LoaderState* stp) case 0: /* Floating point number. * Not generated by the compiler in R16B and later. + * (The literal pool is used instead.) */ - { - Eterm* hp; -#if !defined(ARCH_64) - Uint high, low; -# endif - last_op->a[arg].val = new_literal(stp, &hp, - FLOAT_SIZE_OBJECT); - hp[0] = HEADER_FLONUM; - last_op->a[arg].type = TAG_q; -#if defined(ARCH_64) - GetInt(stp, 8, hp[1]); -# else - GetInt(stp, 4, high); - GetInt(stp, 4, low); - if (must_swap_floats) { - Uint t = high; - high = low; - low = t; - } - hp[1] = high; - hp[2] = low; -# endif - } + LoadError0(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler"); break; case 1: /* List. */ if (arg+1 != arity) { @@ -2286,7 +2315,8 @@ load_code(LoaderState* stp) stp->specific_op = specific; CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */ - code[ci++] = BeamOpCode(stp->specific_op); + last_instr_start = ci + opc[stp->specific_op].adjust; + code[ci++] = BeamOpCodeAddr(stp->specific_op); } /* @@ -2368,7 +2398,8 @@ load_code(LoaderState* stp) break; } break; - case 'd': /* Destination (x(0), x(N), y(N) */ + case 'd': /* Destination (x(N), y(N) */ + case 'S': /* Source (x(N), y(N)) */ switch (tag) { case TAG_x: code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); @@ -2382,11 +2413,29 @@ load_code(LoaderState* stp) break; } break; - case 'I': /* Untagged integer (or pointer). */ - VerifyTag(stp, tag, TAG_u); - code[ci++] = tmp_op->a[arg].val; - break; - case 't': /* Small untagged integer -- can be packed. */ + case 't': /* Small untagged integer (16 bits) -- can be packed. */ + case 'I': /* Untagged integer (32 bits) -- can be packed. */ + case 'W': /* Untagged integer or pointer (machine word). */ +#ifdef DEBUG + switch (*sign) { + case 't': + if (tmp_op->a[arg].val >> 16 != 0) { + load_printf(__LINE__, stp, "value %lu of type 't' does not fit in 16 bits", + tmp_op->a[arg].val); + ASSERT(0); + } + break; +#ifdef ARCH_64 + case 'I': + if (tmp_op->a[arg].val >> 32 != 0) { + load_printf(__LINE__, stp, "value %lu of type 'I' does not fit in 32 bits", + tmp_op->a[arg].val); + ASSERT(0); + } + break; +#endif + } +#endif VerifyTag(stp, tag, TAG_u); code[ci++] = tmp_op->a[arg].val; break; @@ -2396,16 +2445,14 @@ load_code(LoaderState* stp) break; case 'f': /* Destination label */ VerifyTag(stp, tag_to_letter[tag], *sign); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); ci++; break; case 'j': /* 'f' or 'p' */ if (tag == TAG_p) { code[ci] = 0; } else if (tag == TAG_f) { - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); } else { LoadError3(stp, "bad tag %d; expected %d or %d", tag, TAG_f, TAG_p); @@ -2425,7 +2472,6 @@ load_code(LoaderState* stp) LoadError1(stp, "label %d defined more than once", last_label); } stp->labels[last_label].value = ci; - ASSERT(stp->labels[last_label].patches < ci); break; case 'e': /* Export entry */ VerifyTag(stp, tag, TAG_u); @@ -2471,35 +2517,133 @@ load_code(LoaderState* stp) * The packing engine. */ if (opc[stp->specific_op].pack[0]) { - char* prog; /* Program for packing engine. */ - BeamInstr stack[8]; /* Stack. */ - BeamInstr* sp = stack; /* Points to next free position. */ - BeamInstr packed = 0; /* Accumulator for packed operations. */ + char* prog; /* Program for packing engine. */ + struct pack_stack { + BeamInstr instr; + Uint* patch_pos; + } stack[8]; /* Stack. */ + struct pack_stack* sp = stack; /* Points to next free position. */ + BeamInstr packed = 0; /* Accumulator for packed operations. */ + LabelPatch* packed_label = 0; for (prog = opc[stp->specific_op].pack; *prog; prog++) { switch (*prog) { - case 'g': /* Get instruction; push on stack. */ - *sp++ = code[--ci]; - break; + case 'g': /* Get operand and push on stack. */ + ci--; + sp->instr = code[ci]; + sp->patch_pos = 0; + sp++; + break; + case 'f': /* Get possible 'f' operand and push on stack. */ + { + Uint w = code[--ci]; + sp->instr = w; + sp->patch_pos = 0; + + if (w != 0) { + LabelPatch* lbl_p; + int num_patches; + int patch; + + ASSERT(w < stp->num_labels); + lbl_p = stp->labels[w].patches; + num_patches = stp->labels[w].num_patches; + for (patch = num_patches - 1; patch >= 0; patch--) { + if (lbl_p[patch].pos == ci) { + sp->patch_pos = &lbl_p[patch].pos; + break; + } + } + ASSERT(sp->patch_pos); + } + sp++; + } + break; + case 'q': /* Get possible 'q' operand and push on stack. */ + { + LiteralPatch* lp; + + ci--; + sp->instr = code[ci]; + sp->patch_pos = 0; + + for (lp = stp->literal_patches; + lp && lp->pos > ci-MAX_OPARGS; + lp = lp->next) { + if (lp->pos == ci) { + sp->patch_pos = &lp->pos; + break; + } + } + sp++; + } + break; case 'i': /* Initialize packing accumulator. */ packed = code[--ci]; break; case '0': /* Tight shift */ packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci]; + if (packed_label) { + packed_label->packed++; + } break; case '6': /* Shift 16 steps */ packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; + if (packed_label) { + packed_label->packed++; + } break; #ifdef ARCH_64 case 'w': /* Shift 32 steps */ - packed = (packed << BEAM_WIDE_SHIFT) | code[--ci]; - break; + { + Uint w = code[--ci]; + + if (packed_label) { + packed_label->packed++; + } + + /* + * 'w' can handle both labels ('f' and 'j'), as well + * as 'I'. Test whether this is a label. + */ + + if (w < stp->num_labels) { + /* + * Probably a label. Look for patch pointing to this + * position. + */ + LabelPatch* lp = stp->labels[w].patches; + int num_patches = stp->labels[w].num_patches; + int patch; + for (patch = num_patches - 1; patch >= 0; patch--) { + if (lp[patch].pos == ci) { + lp[patch].packed = 1; + packed_label = &lp[patch]; + break; + } + } + } + packed = (packed << BEAM_WIDE_SHIFT) | + (code[ci] & BEAM_WIDE_MASK); + } + break; #endif case 'p': /* Put instruction (from stack). */ - code[ci++] = *--sp; + --sp; + code[ci] = sp->instr; + if (sp->patch_pos) { + *sp->patch_pos = ci; + } + ci++; break; - case 'P': /* Put packed operands. */ - *sp++ = packed; + case 'P': /* Put packed operands (on the stack). */ + sp->instr = packed; + sp->patch_pos = 0; + if (packed_label) { + sp->patch_pos = &packed_label->pos; + packed_label = 0; + } + sp++; packed = 0; break; default: @@ -2513,7 +2657,17 @@ load_code(LoaderState* stp) * Load any list arguments using the primitive tags. */ +#if defined(BEAM_WIDE_SHIFT) + num_trailing_f = 0; +#endif for ( ; arg < tmp_op->arity; arg++) { +#if defined(BEAM_WIDE_SHIFT) + if (tmp_op->a[arg].type == TAG_f) { + num_trailing_f++; + } else { + num_trailing_f = 0; + } +#endif switch (tmp_op->a[arg].type) { case TAG_i: CodeNeed(1); @@ -2527,8 +2681,7 @@ load_code(LoaderState* stp) break; case TAG_f: CodeNeed(1); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; + register_label_patch(stp, tmp_op->a[arg].val, ci, -last_instr_start); ci++; break; case TAG_x: @@ -2554,6 +2707,61 @@ load_code(LoaderState* stp) } } + /* + * If all the extra arguments were 'f' operands, + * and the wordsize is 64 bits, pack two 'f' operands + * into each word. + */ + +#if defined(BEAM_WIDE_SHIFT) + if (num_trailing_f >= 1) { + Uint src_index = ci - num_trailing_f; + Uint src_limit = ci; + Uint dst_limit = src_index + (num_trailing_f+1)/2; + + ci = src_index; + while (ci < dst_limit) { + Uint w[2]; + BeamInstr packed = 0; + int wi; + + w[0] = code[src_index]; + if (src_index+1 < src_limit) { + w[1] = code[src_index+1]; + } else { + w[1] = 0; + } + for (wi = 0; wi < 2; wi++) { + Uint lbl = w[wi]; + LabelPatch* lp = stp->labels[lbl].patches; + int num_patches = stp->labels[lbl].num_patches; + +#if defined(WORDS_BIGENDIAN) + packed <<= BEAM_WIDE_SHIFT; + packed |= lbl & BEAM_WIDE_MASK; +#else + packed >>= BEAM_WIDE_SHIFT; + packed |= lbl << BEAM_WIDE_SHIFT; +#endif + while (num_patches-- > 0) { + if (lp->pos == src_index + wi) { + lp->pos = ci; +#if defined(WORDS_BIGENDIAN) + lp->packed = 2 - wi; +#else + lp->packed = wi + 1; +#endif + break; + } + lp++; + } + } + code[ci++] = packed; + src_index += 2; + } + } +#endif + /* * Handle a few special cases. */ @@ -2600,17 +2808,16 @@ load_code(LoaderState* stp) the size of the ops.tab i_func_info instruction is not the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; - stp->labels[last_label].patches = offset; + register_label_patch(stp, last_label, offset, 0); function_number++; if (stp->arity > MAX_ARG) { LoadError1(stp, "too many arguments: %d", stp->arity); } #ifdef DEBUG - ASSERT(stp->labels[0].patches < 0); /* Should not be referenced. */ + ASSERT(stp->labels[0].num_patches == 0); /* Should not be referenced. */ for (i = 1; i < stp->num_labels; i++) { - ASSERT(stp->labels[i].patches < ci); + ASSERT(stp->labels[i].num_patches <= stp->labels[i].num_allocated); } #endif } @@ -2621,8 +2828,8 @@ load_code(LoaderState* stp) /* Remember offset for the on_load function. */ stp->on_load = ci; break; - case op_bs_put_string_II: - case op_i_bs_match_string_xfII: + case op_bs_put_string_WW: + case op_i_bs_match_string_xfWW: new_string_patch(stp, ci-1); break; @@ -2733,6 +2940,12 @@ load_code(LoaderState* stp) #define never(St) 0 +static int +compiled_with_otp_20_or_higher(LoaderState* stp) +{ + return stp->otp_20_or_higher; +} + /* * Predicate that tests whether a jump table can be used. */ @@ -2872,17 +3085,18 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, op->next = NULL; if (Index.type == TAG_i && Index.val > 0 && + Index.val <= ERTS_MAX_TUPLE_SIZE && (Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[0] = Fail; - op->a[1] = Tuple; + op->a[0] = Tuple; + op->a[1] = Fail; op->a[2].type = TAG_u; op->a[2].val = Index.val; op->a[3] = Dst; } else { op->op = genop_i_element_4; - op->a[0] = Fail; - op->a[1] = Tuple; + op->a[0] = Tuple; + op->a[1] = Fail; op->a[2] = Index; op->a[3] = Dst; } @@ -2962,13 +3176,14 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, op->a[0] = Ms; op->a[1] = Fail; op->a[2] = Dst; +#ifdef ARCH_64 } else if (bits == 32 && (Flags.val & BSF_LITTLE) == 0) { - op->op = genop_i_bs_get_integer_32_4; - op->arity = 4; + op->op = genop_i_bs_get_integer_32_3; + op->arity = 3; op->a[0] = Ms; op->a[1] = Fail; - op->a[2] = Live; - op->a[3] = Dst; + op->a[2] = Dst; +#endif } else { generic: if (bits < SMALL_BITS) { @@ -3103,16 +3318,6 @@ gen_get_binary2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live, } /* - * Predicate to test whether a heap binary should be generated. - */ - -static int -should_gen_heap_bin(LoaderState* stp, GenOpArg Src) -{ - return Src.val <= ERL_ONHEAP_BIN_LIMIT; -} - -/* * Predicate to test whether a binary construction is too big. */ @@ -3384,27 +3589,14 @@ negation_is_small(LoaderState* stp, GenOpArg Int) IS_SSMALL(-((Sint)Int.val)); } - -static int -smp(LoaderState* stp) -{ -#ifdef ERTS_SMP - return 1; -#else - return 0; -#endif -} - /* * Mark this label. */ static int smp_mark_target_label(LoaderState* stp, GenOpArg L) { -#ifdef ERTS_SMP ASSERT(L.type == TAG_f); stp->labels[L.val].looprec_targeted = 1; -#endif return 1; } @@ -3415,12 +3607,8 @@ smp_mark_target_label(LoaderState* stp, GenOpArg L) static int smp_already_locked(LoaderState* stp, GenOpArg L) { -#ifdef ERTS_SMP ASSERT(L.type == TAG_u); return stp->labels[L.val].looprec_targeted; -#else - return 0; -#endif } /* @@ -3434,11 +3622,11 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_i_wait_timeout_2; + op->op = genop_wait_timeout_unlocked_int_2; op->next = NULL; op->arity = 2; - op->a[0] = Fail; - op->a[1].type = TAG_u; + op->a[0].type = TAG_u; + op->a[1] = Fail; if (Time.type == TAG_i && (timeout = Time.val) >= 0 && #if defined(ARCH_64) @@ -3447,7 +3635,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) 1 #endif ) { - op->a[1].val = timeout; + op->a[0].val = timeout; #if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3461,7 +3649,7 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } else { Uint u; (void) term_to_Uint(big, &u); - op->a[1].val = (BeamInstr) u; + op->a[0].val = (BeamInstr) u; } #endif } else { @@ -3481,12 +3669,12 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) Sint timeout; NEW_GENOP(stp, op); - op->op = genop_i_wait_timeout_locked_2; + op->op = genop_wait_timeout_locked_int_2; op->next = NULL; op->arity = 2; - op->a[0] = Fail; - op->a[1].type = TAG_u; - + op->a[0].type = TAG_u; + op->a[1] = Fail; + if (Time.type == TAG_i && (timeout = Time.val) >= 0 && #if defined(ARCH_64) (timeout >> 32) == 0 @@ -3494,7 +3682,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) 1 #endif ) { - op->a[1].val = timeout; + op->a[0].val = timeout; #if !defined(ARCH_64) } else if (Time.type == TAG_q) { Eterm big; @@ -3508,7 +3696,7 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time) } else { Uint u; (void) term_to_Uint(big, &u); - op->a[1].val = (BeamInstr) u; + op->a[0].val = (BeamInstr) u; } #endif } else { @@ -3554,7 +3742,7 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, if (size == 2) { NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_tuple_arity2_6; + op->op = genop_i_select_tuple_arity2_4; GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -3844,14 +4032,13 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, int i, j, align = 0; if (size == 2) { - /* * Use a special-cased instruction if there are only two values. */ NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val2_6; + op->op = genop_i_select_val2_4; GENOP_ARITY(op, arity - 1); op->a[0] = S; op->a[1] = Fail; @@ -3861,47 +4048,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->a[5] = Rest[3]; return op; - - } else if (size > 10) { - - /* binary search instruction */ - - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_val_bins_3; - GENOP_ARITY(op, arity); - op->a[0] = S; - op->a[1] = Fail; - op->a[2].type = TAG_u; - op->a[2].val = size; - for (i = 3; i < arity; i++) { - op->a[i] = Rest[i-3]; - } - - /* - * Sort the values to make them useful for a binary search. - */ - - qsort(op->a+3, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genopargcompare); -#ifdef DEBUG - for (i = 3; i < arity-2; i += 2) { - ASSERT(op->a[i].val < op->a[i+2].val); - } -#endif - return op; } - /* linear search instruction */ - - align = 1; + if (size <= 10) { + /* Use linear search. Reserve place for a sentinel. */ + align = 1; + } arity += 2*align; size += align; NEW_GENOP(stp, op); op->next = NULL; - op->op = genop_i_select_val_lins_3; + op->op = (align == 0) ? genop_i_select_val_bins_3 : genop_i_select_val_lins_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -3915,7 +4074,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, } /* - * Sort the values to make them useful for a sentinel search + * Sort the values to make them useful for a binary or sentinel search. */ qsort(tmp, size - align, 2*sizeof(GenOpArg), @@ -3930,11 +4089,12 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, erts_free(ERTS_ALC_T_LOADER_TMP, (void *) tmp); - /* add sentinel */ - - op->a[j].type = TAG_u; - op->a[j].val = ~((BeamInstr)0); - op->a[j+size] = Fail; + if (align) { + /* Add sentinel for linear search. */ + op->a[j].type = TAG_u; + op->a[j].val = ~((BeamInstr)0); + op->a[j+size] = Fail; + } #ifdef DEBUG for (i = 0; i < size - 1; i++) { @@ -4222,6 +4382,92 @@ literal_is_map(LoaderState* stp, GenOpArg Lit) } /* + * Predicate to test whether all of the given new small map keys are literals + */ +static int +is_small_map_literal_keys(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +{ + if (Size.val > MAP_SMALL_MAP_LIMIT) { + return 0; + } + + /* + * Operations with non-literals have always only one key. + */ + if (Size.val != 2) { + return 1; + } + + switch (Rest[0].type) { + case TAG_a: + case TAG_i: + case TAG_n: + case TAG_q: + return 1; + default: + return 0; + } +} + +static GenOp* +gen_new_small_map_lit(LoaderState* stp, GenOpArg Dst, GenOpArg Live, + GenOpArg Size, GenOpArg* Rest) +{ + unsigned size = Size.val; + Uint lit; + unsigned i; + GenOp* op; + GenOpArg* dst; + Eterm* hp; + Eterm* tmp; + Eterm* thp; + Eterm keys; + + NEW_GENOP(stp, op); + GENOP_ARITY(op, 3 + size/2); + op->next = NULL; + op->op = genop_i_new_small_map_lit_3; + + tmp = thp = erts_alloc(ERTS_ALC_T_LOADER_TMP, (1 + size/2) * sizeof(*tmp)); + keys = make_tuple(thp); + *thp++ = make_arityval(size/2); + + dst = op->a+3; + + for (i = 0; i < size; i += 2) { + switch (Rest[i].type) { + case TAG_a: + *thp++ = Rest[i].val; + ASSERT(is_atom(Rest[i].val)); + break; + case TAG_i: + *thp++ = make_small(Rest[i].val); + break; + case TAG_n: + *thp++ = NIL; + break; + case TAG_q: + *thp++ = stp->literals[Rest[i].val].term; + break; + } + *dst++ = Rest[i + 1]; + } + + if (!find_literal(stp, keys, &lit)) { + lit = new_literal(stp, &hp, 1 + size/2); + sys_memcpy(hp, tmp, (1 + size/2) * sizeof(*tmp)); + } + erts_free(ERTS_ALC_T_LOADER_TMP, tmp); + + op->a[0] = Dst; + op->a[1] = Live; + op->a[2].type = TAG_q; + op->a[2].val = lit; + + return op; +} + +/* * Predicate to test whether the given literal is an empty map. */ @@ -4732,21 +4978,57 @@ freeze_code(LoaderState* stp) */ for (i = 0; i < stp->num_labels; i++) { - Sint this_patch; - Sint next_patch; + Uint patch; Uint value = stp->labels[i].value; - - if (value == 0 && stp->labels[i].patches >= 0) { + + if (value == 0 && stp->labels[i].num_patches != 0) { LoadError1(stp, "label %d not resolved", i); } ASSERT(value < stp->ci); - this_patch = stp->labels[i].patches; - while (this_patch >= 0) { - ASSERT(this_patch < stp->ci); - next_patch = codev[this_patch]; - ASSERT(next_patch < stp->ci); - codev[this_patch] = (BeamInstr) (codev + value); - this_patch = next_patch; + for (patch = 0; patch < stp->labels[i].num_patches; patch++) { + LabelPatch* lp = &stp->labels[i].patches[patch]; + Uint pos = lp->pos; + ASSERT(pos < stp->ci); + if (pos < stp->num_functions) { + /* + * This is the array of pointers to the beginning of + * each function. The pointers must remain absolute. + */ + codev[pos] = (BeamInstr) (codev + value); + } else { +#ifdef DEBUG + Uint w; +#endif + Sint32 rel = lp->offset + value; + switch (lp->packed) { + case 0: /* Not packed */ + ASSERT(codev[pos] == i); + codev[pos] = rel; + break; +#ifdef BEAM_WIDE_MASK + case 1: /* Least significant word. */ +#ifdef DEBUG + w = codev[pos] & BEAM_WIDE_MASK; + /* Correct label in least significant word? */ + ASSERT(w == i); +#endif + codev[pos] = (codev[pos] & ~BEAM_WIDE_MASK) | + (rel & BEAM_WIDE_MASK); + break; + case 2: /* Most significant word */ +#ifdef DEBUG + w = (codev[pos] >> BEAM_WIDE_SHIFT) & BEAM_WIDE_MASK; + /* Correct label in most significant word? */ + ASSERT(w == i); +#endif + codev[pos] = ((Uint)rel << BEAM_WIDE_SHIFT) | + (codev[pos] & BEAM_WIDE_MASK); + break; +#endif + default: + ASSERT(0); + } + } } } CHKBLK(ERTS_ALC_T_CODE,code_hdr); @@ -4789,8 +5071,11 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) catches = BEAM_CATCHES_NIL; while (index != 0) { BeamInstr next = codev[index]; - codev[index] = BeamOpCode(op_catch_yf); - catches = beam_catches_cons((BeamInstr *)codev[index+2], catches); + BeamInstr* abs_addr; + codev[index] = BeamOpCodeAddr(op_catch_yf); + /* We must make the address of the label absolute again. */ + abs_addr = (BeamInstr *)codev + index + codev[index+2]; + catches = beam_catches_cons(abs_addr, catches); codev[index+2] = make_catch(catches); index = next; } @@ -4861,7 +5146,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) /* * We are hiding a pointer into older code. */ - erts_smp_refc_dec(&fe->refc, 1); + erts_refc_dec(&fe->refc, 1); } fe->address = code_ptr; #ifdef HIPE @@ -5257,12 +5542,15 @@ get_tag_and_value(LoaderState* stp, Uint len_code, { Uint count; Sint val; - byte default_buf[128]; - byte* bigbuf = default_buf; + byte default_byte_buf[128]; + byte* byte_buf = default_byte_buf; + Eterm default_big_buf[128/sizeof(Eterm)]; + Eterm* big_buf = default_big_buf; + Eterm tmp_big; byte* s; int i; int neg = 0; - Uint arity; + Uint words_needed; Eterm* hp; /* @@ -5339,8 +5627,11 @@ get_tag_and_value(LoaderState* stp, Uint len_code, *result = val; return TAG_i; } else { - *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE); - (void) small_to_big(val, hp); + tmp_big = small_to_big(val, big_buf); + if (!find_literal(stp, tmp_big, result)) { + *result = new_literal(stp, &hp, BIG_UINT_HEAP_SIZE); + sys_memcpy(hp, big_buf, BIG_UINT_HEAP_SIZE*sizeof(Eterm)); + } return TAG_q; } } @@ -5350,8 +5641,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code, * (including margin). */ - if (count+8 > sizeof(default_buf)) { - bigbuf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8); + if (count+8 > sizeof(default_byte_buf)) { + byte_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, count+8); } /* @@ -5360,20 +5651,20 @@ get_tag_and_value(LoaderState* stp, Uint len_code, GetString(stp, s, count); for (i = 0; i < count; i++) { - bigbuf[count-i-1] = *s++; + byte_buf[count-i-1] = *s++; } /* * Check if the number is negative, and negate it if so. */ - if ((bigbuf[count-1] & 0x80) != 0) { + if ((byte_buf[count-1] & 0x80) != 0) { unsigned carry = 1; neg = 1; for (i = 0; i < count; i++) { - bigbuf[i] = ~bigbuf[i] + carry; - carry = (bigbuf[i] == 0 && carry == 1); + byte_buf[i] = ~byte_buf[i] + carry; + carry = (byte_buf[i] == 0 && carry == 1); } ASSERT(carry == 0); } @@ -5382,33 +5673,52 @@ get_tag_and_value(LoaderState* stp, Uint len_code, * Align to word boundary. */ - if (bigbuf[count-1] == 0) { + if (byte_buf[count-1] == 0) { count--; } - if (bigbuf[count-1] == 0) { + if (byte_buf[count-1] == 0) { LoadError0(stp, "bignum not normalized"); } while (count % sizeof(Eterm) != 0) { - bigbuf[count++] = 0; + byte_buf[count++] = 0; } /* - * Allocate heap space for the bignum and copy it. + * Convert to a bignum. */ - arity = count/sizeof(Eterm); - *result = new_literal(stp, &hp, arity+1); - if (is_nil(bytes_to_big(bigbuf, count, neg, hp))) - goto load_error; + words_needed = count/sizeof(Eterm) + 1; + if (words_needed*sizeof(Eterm) > sizeof(default_big_buf)) { + big_buf = erts_alloc(ERTS_ALC_T_LOADER_TMP, words_needed*sizeof(Eterm)); + } + tmp_big = bytes_to_big(byte_buf, count, neg, big_buf); + if (is_nil(tmp_big)) { + goto load_error; + } - if (bigbuf != default_buf) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); + /* + * Create a literal if there is no previous literal with the same value. + */ + + if (!find_literal(stp, tmp_big, result)) { + *result = new_literal(stp, &hp, words_needed); + sys_memcpy(hp, big_buf, words_needed*sizeof(Eterm)); + } + + if (byte_buf != default_byte_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf); + } + if (big_buf != default_big_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf); } return TAG_q; load_error: - if (bigbuf != default_buf) { - erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf); + if (byte_buf != default_byte_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) byte_buf); + } + if (big_buf != default_big_buf) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) big_buf); } return -1; } @@ -5453,8 +5763,7 @@ new_label(LoaderState* stp) stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, (void *) stp->labels, stp->num_labels * sizeof(Label)); - stp->labels[num].value = 0; - stp->labels[num].patches = -1; + init_label(&stp->labels[num]); return num; } @@ -5509,6 +5818,24 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) return stp->num_literals++; } +static int +find_literal(LoaderState* stp, Eterm needle, Uint *idx) +{ + int i; + + /* + * The search is done backwards since the most recent literals + * allocated by the loader itself will be placed at the end + */ + for (i = stp->num_literals - 1; i >= 0; i--) { + if (EQ(needle, stp->literals[i].term)) { + *idx = (Uint) i; + return 1; + } + } + return 0; +} + Eterm erts_module_info_0(Process* p, Eterm module) { @@ -5699,9 +6026,9 @@ int erts_is_function_native(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); + return BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call) || + BeamIsOpCode(erts_codeinfo_to_code(ci)[0], op_hipe_trap_call_closure); #else return 0; #endif @@ -5770,7 +6097,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */ Eterm tuple; if (ep->addressv[code_ix] == ep->beam && - ep->beam[0] == (BeamInstr) em_call_error_handler) { + BeamIsOpCode(ep->beam[0], op_call_error_handler)) { /* There is a call to the function, but it does not exist. */ continue; } @@ -6041,7 +6368,7 @@ make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, Be { DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); ASSERT(WORDS_PER_FUNCTION == 6); - info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->op = BeamOpCodeAddr(op_i_func_info_IaaI); info->u.ncallee = (void (*)(void)) native; info->mfa.module = mod; info->mfa.function = func; @@ -6146,7 +6473,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { - *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + *erts_codeinfo_to_code(ci) = BeamOpCodeAddr(op_hipe_trap_call_closure); fe->address = erts_codeinfo_to_code(ci); } } @@ -6276,7 +6603,7 @@ patch_funentries(Eterm Patchlist) fe = erts_get_fun_entry(Mod, uniq, index); fe->native_address = (Uint *)native_address; - erts_smp_refc_dec(&fe->refc, 1); + erts_refc_dec(&fe->refc, 1); if (!patch(Addresses, (Uint) fe)) return 0; @@ -6470,7 +6797,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) * as the body until we know what kind of trap we should put there. */ code_hdr->functions[i] = (ErtsCodeInfo*)fp; - op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ + op = BeamOpCodeAddr(op_hipe_trap_call); /* Might be changed later. */ fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, (Uint)native_address, op); } @@ -6480,7 +6807,7 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) */ code_hdr->functions[i] = (ErtsCodeInfo*)fp; - *fp++ = (BeamInstr) BeamOp(op_int_code_end); + *fp++ = BeamOpCodeAddr(op_int_code_end); /* * Copy attributes and compilation information. diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index c088bdb751..156c3c45e2 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -37,11 +37,6 @@ typedef struct gen_op_entry { extern const GenOpEntry gen_opc[]; -extern BeamInstr beam_debug_apply[]; -extern BeamInstr* em_call_error_handler; -extern BeamInstr* em_apply_bif; -extern BeamInstr* em_call_nif; - struct ErtsLiteralArea_; /* @@ -111,11 +106,7 @@ typedef struct beam_code_header { }BeamCodeHeader; -#ifdef ERTS_DIRTY_SCHEDULERS # define BEAM_NIF_MIN_FUNC_SZ 4 -#else -# define BEAM_NIF_MIN_FUNC_SZ 3 -#endif void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 9b0335e83d..6e373a3480 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -29,12 +29,12 @@ typedef struct { BeamInstr* start; /* Pointer to start of module. */ - erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */ + erts_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */ } Range; /* Range 'end' needs to be atomic as we purge module by setting end=start in active code_ix */ -#define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) +#define RANGE_END(R) ((BeamInstr*)erts_atomic_read_nob(&(R)->end)) static Range* find_range(BeamInstr* pc); static void lookup_loc(FunctionInfo* fi, const BeamInstr* pc, @@ -49,10 +49,10 @@ struct ranges { Range* modules; /* Sorted lists of module addresses. */ Sint n; /* Number of range entries. */ Sint allocated; /* Number of allocated entries. */ - erts_smp_atomic_t mid; /* Cached search start point */ + erts_atomic_t mid; /* Cached search start point */ }; static struct ranges r[ERTS_NUM_CODE_IX]; -static erts_smp_atomic_t mem_used; +static erts_atomic_t mem_used; static Range* write_ptr; #ifdef HARD_DEBUG @@ -90,12 +90,12 @@ erts_init_ranges(void) { Sint i; - erts_smp_atomic_init_nob(&mem_used, 0); + erts_atomic_init_nob(&mem_used, 0); for (i = 0; i < ERTS_NUM_CODE_IX; i++) { r[i].modules = 0; r[i].n = 0; r[i].allocated = 0; - erts_smp_atomic_init_nob(&r[i].mid, 0); + erts_atomic_init_nob(&r[i].mid, 0); } } @@ -107,12 +107,12 @@ erts_start_staging_ranges(int num_new) Sint need; if (r[dst].modules) { - erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated); + erts_atomic_add_nob(&mem_used, -r[dst].allocated); erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules); } need = r[dst].allocated = r[src].n + num_new; - erts_smp_atomic_add_nob(&mem_used, need); + erts_atomic_add_nob(&mem_used, need); write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS, need * sizeof(Range)); r[dst].modules = write_ptr; @@ -135,7 +135,7 @@ erts_end_staging_ranges(int commit) if (rp->start < RANGE_END(rp)) { /* Only insert a module that has not been purged. */ write_ptr->start = rp->start; - erts_smp_atomic_init_nob(&write_ptr->end, + erts_atomic_init_nob(&write_ptr->end, (erts_aint_t)(RANGE_END(rp))); write_ptr++; } @@ -161,7 +161,7 @@ erts_end_staging_ranges(int commit) } r[dst].modules = mp; CHECK(&r[dst]); - erts_smp_atomic_set_nob(&r[dst].mid, + erts_atomic_set_nob(&r[dst].mid, (erts_aint_t) (r[dst].modules + r[dst].n / 2)); } @@ -182,7 +182,7 @@ erts_update_ranges(BeamInstr* code, Uint size) */ if (r[dst].modules == NULL) { Sint need = 128; - erts_smp_atomic_add_nob(&mem_used, need); + erts_atomic_add_nob(&mem_used, need); r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, need * sizeof(Range)); r[dst].allocated = need; @@ -192,7 +192,7 @@ erts_update_ranges(BeamInstr* code, Uint size) ASSERT(r[dst].modules); write_ptr->start = code; - erts_smp_atomic_init_nob(&(write_ptr->end), + erts_atomic_init_nob(&(write_ptr->end), (erts_aint_t)(((byte *)code) + size)); write_ptr++; } @@ -201,13 +201,13 @@ void erts_remove_from_ranges(BeamInstr* code) { Range* rp = find_range(code); - erts_smp_atomic_set_nob(&rp->end, (erts_aint_t)rp->start); + erts_atomic_set_nob(&rp->end, (erts_aint_t)rp->start); } UWord erts_ranges_sz(void) { - return erts_smp_atomic_read_nob(&mem_used) * sizeof(Range); + return erts_atomic_read_nob(&mem_used) * sizeof(Range); } /* @@ -262,7 +262,7 @@ find_range(BeamInstr* pc) ErtsCodeIndex active = erts_active_code_ix(); Range* low = r[active].modules; Range* high = low + r[active].n; - Range* mid = (Range *) erts_smp_atomic_read_nob(&r[active].mid); + Range* mid = (Range *) erts_atomic_read_nob(&r[active].mid); CHECK(&r[active]); while (low < high) { @@ -271,7 +271,7 @@ find_range(BeamInstr* pc) } else if (pc >= RANGE_END(mid)) { low = mid + 1; } else { - erts_smp_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); + erts_atomic_set_nob(&r[active].mid, (erts_aint_t) mid); return mid; } mid = low + (high-low) / 2; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 40dd4129d2..ad555bb195 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -57,13 +57,10 @@ static Export dsend_continue_trap_export; Export *erts_convert_time_unit_trap = NULL; static Export *await_msacc_mod_trap = NULL; -static erts_smp_atomic32_t msacc; +static erts_atomic32_t msacc; static Export *await_sched_wall_time_mod_trap; -static erts_smp_atomic32_t sched_wall_time; - -static erts_smp_mtx_t ports_snapshot_mtx; -erts_smp_atomic_t erts_dead_ports_ptr; /* To store dying ports during snapshot */ +static erts_atomic32_t sched_wall_time; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -101,14 +98,12 @@ static int insert_internal_link(Process* p, Eterm rpid) ASSERT(is_internal_pid(rpid)); -#ifdef ERTS_SMP if (IS_TRACED(p) && (ERTS_TRACE_FLAGS(p) & (F_TRACE_SOL|F_TRACE_SOL1))) { rp_locks = ERTS_PROC_LOCKS_ALL; } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); -#endif + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); /* get a pointer to the process struct of the linked process */ rp = erts_pid2proc_opt(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, @@ -116,7 +111,7 @@ static int insert_internal_link(Process* p, Eterm rpid) ERTS_P2P_FLG_ALLOW_OTHER_X); if (!rp) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); return 0; } @@ -142,10 +137,10 @@ static int insert_internal_link(Process* p, Eterm rpid) rp, am_getting_linked, p->common.id); if (p == rp) - erts_smp_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(p, rp_locks & ~ERTS_PROC_LOCK_MAIN); else { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, rp_locks); } return 1; @@ -181,13 +176,13 @@ BIF_RETTYPE link_1(BIF_ALIST_1) goto res_no_proc; } - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); if (erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1) >= 0) send_link_signal = 1; /* else: already linked */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); if (send_link_signal) { Eterm ref; @@ -215,11 +210,11 @@ BIF_RETTYPE link_1(BIF_ALIST_1) if (is_external_pid(BIF_ARG_1)) { - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); /* We may earn time by checking first that we're not linked already */ if (erts_lookup_link(ERTS_P_LINKS(BIF_P), BIF_ARG_1) != NULL) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); } else { @@ -228,7 +223,7 @@ BIF_RETTYPE link_1(BIF_ALIST_1) ErtsDSigData dsd; dep = external_pid_dist_entry(BIF_ARG_1); if (dep == erts_this_dist_entry) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); goto res_no_proc; } @@ -237,13 +232,13 @@ BIF_RETTYPE link_1(BIF_ALIST_1) case ERTS_DSIG_PREP_NOT_ALIVE: /* Let the dlink trap handle it */ case ERTS_DSIG_PREP_NOT_CONNECTED: - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); BIF_TRAP1(dlink_trap, BIF_P, BIF_ARG_1); case ERTS_DSIG_PREP_CONNECTED: /* We are connected. Setup link and send link signal */ - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); erts_add_link(&ERTS_P_LINKS(BIF_P), LINK_PID, BIF_ARG_1); lnk = erts_add_or_lookup_link(&(dep->nlinks), @@ -252,9 +247,9 @@ BIF_RETTYPE link_1(BIF_ALIST_1) ASSERT(lnk != NULL); erts_add_link(&ERTS_LINK_ROOT(lnk), LINK_PID, BIF_ARG_1); - erts_smp_de_links_unlock(dep); - erts_smp_de_runlock(dep); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_de_runlock(dep); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); code = erts_dsig_send_link(&dsd, BIF_P->common.id, BIF_ARG_1); if (code == ERTS_DSIG_SEND_YIELD) @@ -270,11 +265,11 @@ BIF_RETTYPE link_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); res_no_proc: { - erts_aint32_t state = erts_smp_atomic32_read_nob(&BIF_P->state); + erts_aint32_t state = erts_atomic32_read_nob(&BIF_P->state); if (state & ERTS_PSFLG_TRAP_EXIT) { ErtsProcLocks locks = ERTS_PROC_LOCK_MAIN; erts_deliver_exit_message(BIF_ARG_1, BIF_P, &locks, am_noproc, NIL); - erts_smp_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); + erts_proc_unlock(BIF_P, ~ERTS_PROC_LOCK_MAIN & locks); BIF_RET(am_true); } else @@ -293,58 +288,41 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) ErtsMonitor *mon; int code; Eterm res = am_false; -#ifndef ERTS_SMP - int stale_mon = 0; -#endif - ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK) + ERTS_LC_ASSERT((ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK) == erts_proc_lc_my_proc_locks(c_p)); code = erts_dsig_prepare(&dsd, dep, c_p, ERTS_DSP_RLOCK, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: case ERTS_DSIG_PREP_NOT_CONNECTED: -#ifndef ERTS_SMP - /* XXX Is this possible? Shouldn't this link - previously have been removed if the node - had previously been disconnected. */ - ASSERT(0); - stale_mon = 1; -#endif /* * In the smp case this is possible if the node goes * down just before the call to demonitor. */ if (dep) { - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); dmon = erts_remove_monitor(&dep->monitors, ref); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (dmon) erts_destroy_monitor(dmon); } mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); res = am_true; break; case ERTS_DSIG_PREP_CONNECTED: - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); dmon = erts_remove_monitor(&dep->monitors, ref); - erts_smp_de_links_unlock(dep); - erts_smp_de_runlock(dep); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_de_runlock(dep); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); if (!dmon) { -#ifndef ERTS_SMP - /* XXX How is this possible? Shouldn't this link - previously have been removed when the distributed - end was removed. */ - ASSERT(0); - stale_mon = 1; -#endif /* * This is possible when smp support is enabled. * 'DOWN' message just arrived. @@ -373,18 +351,6 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) return am_internal_error; } -#ifndef ERTS_SMP - if (stale_mon) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Stale process monitor %T to ", ref); - if (is_atom(to)) - erts_dsprintf(dsbufp, "{%T, %T}", to, dep->sysname); - else - erts_dsprintf(dsbufp, "%T", to); - erts_dsprintf(dsbufp, " found\n"); - erts_send_error_to_logger(c_p->group_leader, dsbufp); - } -#endif /* * We aren't allowed to destroy 'mon' until now, since 'to' @@ -394,7 +360,7 @@ remote_demonitor(Process *c_p, DistEntry *dep, Eterm ref, Eterm to) lookup and remove */ erts_destroy_monitor(mon); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); return res; } @@ -408,13 +374,9 @@ demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res) ERTS_P2P_FLG_ALLOW_OTHER_X); ErtsMonitor *mon = erts_remove_monitor(&ERTS_P_MONITORS(c_p), ref); -#ifndef ERTS_SMP - ASSERT(mon); -#else if (!mon) *res = am_false; else -#endif { *res = am_true; erts_destroy_monitor(mon); @@ -423,12 +385,12 @@ demonitor_local_process(Process *c_p, Eterm ref, Eterm to, Eterm *res) ErtsMonitor *rmon; rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); if (rp != c_p) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon != NULL) erts_destroy_monitor(rmon); } else { - ERTS_SMP_ASSERT_IS_NOT_EXITING(c_p); + ERTS_ASSERT_IS_NOT_EXITING(c_p); } } @@ -441,7 +403,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target) if (!port) { BIF_ERROR(origin, BADARG); } - erts_smp_proc_unlock(origin, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(origin, ERTS_PROC_LOCK_LINK); if (port) { Eterm trap_ref; @@ -461,7 +423,7 @@ demonitor_local_port(Process *origin, Eterm ref, Eterm target) } } else { - ERTS_SMP_ASSERT_IS_NOT_EXITING(origin); + ERTS_ASSERT_IS_NOT_EXITING(origin); } BIF_RET(res); } @@ -475,11 +437,10 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) ErtsMonitor *mon = NULL; /* The monitor entry to delete */ Eterm to = NIL; /* Monitor link traget */ DistEntry *dep = NULL; /* Target's distribution entry */ - int deref_de = 0; BIF_RETTYPE res = am_false; int unlock_link = 1; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_LINK); + erts_proc_lock(c_p, ERTS_PROC_LOCK_LINK); if (is_not_internal_ref(ref)) { res = am_badarg; @@ -505,8 +466,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) ASSERT(is_node_name_atom(to)); dep = erts_sysname_to_connected_dist_entry(to); ASSERT(dep != erts_this_dist_entry); - if (dep) - deref_de = 1; } else if (is_port(to)) { if (port_dist_entry(to) != erts_this_dist_entry) { goto badarg; @@ -524,11 +483,6 @@ BIF_RETTYPE demonitor(Process *c_p, Eterm ref, Eterm *multip) unlock_link = 0; } else { /* Local monitor */ - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); - } - dep = NULL; demonitor_local_process(c_p, ref, to, &res); } break; @@ -541,14 +495,9 @@ badarg: done: if (unlock_link) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); - - if (deref_de) { - ASSERT(dep); - erts_deref_dist_entry(dep); - } + erts_proc_unlock(c_p, ERTS_PROC_LOCK_LINK); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); BIF_RET(res); } @@ -671,12 +620,12 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) return ret; } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); rp = erts_pid2proc_opt(p, p_locks, target, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); if (!rp) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; if (boolean) ret = am_false; @@ -693,10 +642,10 @@ local_pid_monitor(Process *p, Eterm target, Eterm mon_ref, int boolean) erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, target, NIL); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, mon_ref, p->common.id, NIL); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } - erts_smp_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(p, p_locks & ~ERTS_PROC_LOCK_MAIN); return ret; } @@ -730,7 +679,7 @@ res_no_proc: break; } } - erts_smp_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(origin, p_locks & ~ERTS_PROC_LOCK_MAIN); BIF_RET(ref); } @@ -745,7 +694,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name) Process *proc = NULL; Port *port = NULL; - erts_smp_proc_lock(self, ERTS_PROC_LOCK_LINK); + erts_proc_lock(self, ERTS_PROC_LOCK_LINK); erts_whereis_name(self, p_locks, target_name, &proc, ERTS_PROC_LOCK_LINK, @@ -764,7 +713,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name) Eterm item; UseTmpHeap(3,self); - erts_smp_proc_unlock(self, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(self, ERTS_PROC_LOCK_LINK); p_locks &= ~ERTS_PROC_LOCK_LINK; item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname); @@ -775,7 +724,7 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name) UnUseTmpHeap(3,self); } else if (port) { - erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); p_locks &= ~ERTS_PROC_LOCK_MAIN; switch (erts_port_monitor(self, port, target_name, &ret)) { @@ -796,16 +745,16 @@ local_name_monitor(Process *self, Eterm type, Eterm target_name) proc->common.id, target_name); erts_add_monitor(&ERTS_P_MONITORS(proc), MON_TARGET, ret, self->common.id, target_name); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(proc, ERTS_PROC_LOCK_LINK); } if (p_locks) { - erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); } BIF_RET(ret); badarg: if (p_locks) { - erts_smp_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(self, p_locks & ~ERTS_PROC_LOCK_MAIN); } BIF_ERROR(self, BADARG); } @@ -818,20 +767,20 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, BIF_RETTYPE ret; int code; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_RLOCK, 0); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: /* Let the dmonitor_p trap handle it */ case ERTS_DSIG_PREP_NOT_CONNECTED: - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); ERTS_BIF_PREP_TRAP2(ret, dmonitor_p_trap, p, bifarg1, bifarg2); break; case ERTS_DSIG_PREP_CONNECTED: if (!(dep->flags & DFLAG_DIST_MONITOR) || (byname && !(dep->flags & DFLAG_DIST_MONITOR_NAME))) { - erts_smp_de_runlock(dep); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_de_runlock(dep); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); ERTS_BIF_PREP_ERROR(ret, p, BADARG); } else { @@ -850,16 +799,16 @@ remote_monitor(Process *p, Eterm bifarg1, Eterm bifarg2, d_name = NIL; } - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); erts_add_monitor(&ERTS_P_MONITORS(p), MON_ORIGIN, mon_ref, p_trgt, p_name); erts_add_monitor(&(dep->monitors), MON_TARGET, mon_ref, p->common.id, d_name); - erts_smp_de_links_unlock(dep); - erts_smp_de_runlock(dep); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_de_runlock(dep); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); code = erts_dsig_send_monitor(&dsd, p->common.id, target, mon_ref); if (code == ERTS_DSIG_SEND_YIELD) @@ -882,7 +831,6 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) Eterm target = BIF_ARG_2; BIF_RETTYPE ret; DistEntry *dep = NULL; - int deref_de = 0; /* Only process monitors are implemented */ switch (BIF_ARG_1) { @@ -892,10 +840,10 @@ BIF_RETTYPE monitor_2(BIF_ALIST_2) goto badarg; } ref = erts_make_ref(BIF_P); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); erts_add_monitor(&ERTS_P_MONITORS(BIF_P), MON_TIME_OFFSET, ref, am_clock_service, NIL); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); erts_monitor_time_offset(BIF_P->common.id, ref); BIF_RET(ref); } @@ -942,21 +890,14 @@ local_port: } dep = erts_sysname_to_connected_dist_entry(remote_node); if (dep == erts_this_dist_entry) { - deref_de = 1; ret = local_name_monitor(BIF_P, BIF_ARG_1, name); } else { - if (dep) - deref_de = 1; ret = remote_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2, dep, name, 1); } } else { badarg: ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); } - if (deref_de) { - deref_de = 0; - erts_deref_dist_entry(dep); - } return ret; } @@ -1012,7 +953,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.max_heap_size = H_MAX_SIZE; so.max_heap_flags = H_MAX_FLAGS; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -1153,15 +1094,13 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) } if (is_internal_port(BIF_ARG_1)) { - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); -#ifdef ERTS_SMP + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); if (ERTS_PROC_PENDING_EXIT(BIF_P)) goto handle_pending_exit; -#endif l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); if (l) { Port *prt; @@ -1203,14 +1142,12 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) /* Blind removal, we might have trapped or anything, this leaves us in a state where monitors might be inconsistent, but the dist code should take care of it. */ - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); -#ifdef ERTS_SMP + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); if (ERTS_PROC_PENDING_EXIT(BIF_P)) goto handle_pending_exit; -#endif l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); - erts_smp_proc_unlock(BIF_P, + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); if (l) @@ -1252,7 +1189,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) /* Internal pid... */ - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); cp_locks |= ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS; @@ -1261,13 +1198,11 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) BIF_ARG_1, ERTS_PROC_LOCK_LINK, ERTS_P2P_FLG_ALLOW_OTHER_X); -#ifdef ERTS_SMP if (ERTS_PROC_PENDING_EXIT(BIF_P)) { if (rp && rp != BIF_P) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); goto handle_pending_exit; } -#endif /* unlink and ignore errors */ l = erts_remove_link(&ERTS_P_LINKS(BIF_P), BIF_ARG_1); @@ -1275,7 +1210,7 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) erts_destroy_link(l); if (!rp) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); + ERTS_ASSERT_IS_NOT_EXITING(BIF_P); } else { rl = erts_remove_link(&ERTS_P_LINKS(rp), BIF_P->common.id); @@ -1283,29 +1218,27 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) erts_destroy_link(rl); if (IS_TRACED_FL(rp, F_TRACE_PROCS) && rl != NULL) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_STATUS); cp_locks &= ~ERTS_PROC_LOCK_STATUS; trace_proc(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK), rp, am_getting_unlinked, BIF_P->common.id); } if (rp != BIF_P) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } - erts_smp_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, cp_locks & ~ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); -#ifdef ERTS_SMP handle_pending_exit: erts_handle_pending_exit(BIF_P, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_LINK | ERTS_PROC_LOCK_STATUS)); ASSERT(ERTS_PROC_IS_EXITING(BIF_P)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); ERTS_BIF_EXITED(BIF_P); -#endif } BIF_RETTYPE hibernate_3(BIF_ALIST_3) @@ -1317,7 +1250,11 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3) */ Eterm reg[3]; - if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { + reg[0] = BIF_ARG_1; + reg[1] = BIF_ARG_2; + reg[2] = BIF_ARG_3; + + if (erts_hibernate(BIF_P, reg)) { /* * If hibernate succeeded, TRAP. The process will be wait in a * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE); @@ -1638,7 +1575,7 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) if (BIF_ARG_1 == BIF_P->common.id) { rp_locks = ERTS_PROC_LOCKS_ALL; rp = BIF_P; - erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(rp, ERTS_PROC_LOCKS_ALL_MINOR); } else { rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; @@ -1660,12 +1597,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) NIL, NULL, BIF_P == rp ? ERTS_XSIG_FLG_NO_IGN_NORMAL : 0); -#ifdef ERTS_SMP if (rp == BIF_P) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); -#endif + erts_proc_unlock(rp, rp_locks); /* * We may have exited ourselves and may have to take action. */ @@ -1777,21 +1712,19 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) * true. For more info, see implementation of * erts_send_exit_signal(). */ - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); if (trap_exit) - state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, + state = erts_atomic32_read_bor_mb(&BIF_P->state, ERTS_PSFLG_TRAP_EXIT); else - state = erts_smp_atomic32_read_band_mb(&BIF_P->state, + state = erts_atomic32_read_band_mb(&BIF_P->state, ~ERTS_PSFLG_TRAP_EXIT); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_XSIG_SEND); -#ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); ERTS_BIF_EXITED(BIF_P); } -#endif old_value = (state & ERTS_PSFLG_TRAP_EXIT) ? am_true : am_false; BIF_RET(old_value); @@ -1809,15 +1742,13 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) if (sched == 0) { new = NULL; - state = erts_smp_atomic32_read_band_mb(&BIF_P->state, + state = erts_atomic32_read_band_mb(&BIF_P->state, ~ERTS_PSFLG_BOUND); } else { new = erts_schedid2runq(sched); -#ifdef ERTS_SMP erts_atomic_set_nob(&BIF_P->run_queue, (erts_aint_t) new); -#endif - state = erts_smp_atomic32_read_bor_mb(&BIF_P->state, + state = erts_atomic32_read_bor_mb(&BIF_P->state, ERTS_PSFLG_BOUND); } @@ -1898,7 +1829,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } else { goto error; } - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); old_value = (ERTS_TRACE_FLAGS(BIF_P) & F_SENSITIVE ? am_true : am_false); @@ -1907,7 +1838,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } else { ERTS_TRACE_FLAGS(BIF_P) &= ~F_SENSITIVE; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); /* make sure to bump all reds so that we get rescheduled immediately so setting takes effect */ BIF_RET2(old_value, CONTEXT_REDS); @@ -1949,15 +1880,11 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3) Process *rp; Eterm res; -#ifdef ERTS_SMP rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, BIF_ARG_1, ERTS_PROC_LOCK_MAIN); if (rp == ERTS_PROC_LOCK_BUSY) ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); -#else - rp = erts_proc_lookup(BIF_ARG_1); -#endif if (!rp) BIF_ERROR(BIF_P, BADARG); @@ -1965,7 +1892,7 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3) res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3); if (rp != BIF_P) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); return res; } @@ -2056,6 +1983,7 @@ static Sint remote_send(Process *p, DistEntry *dep, ASSERT(is_atom(to) || is_external_pid(to)); + ctx->dep = dep; code = erts_dsig_prepare(&ctx->dsd, dep, p, ERTS_DSP_NO_LOCK, !ctx->suspend); switch (code) { case ERTS_DSIG_PREP_NOT_ALIVE: @@ -2257,7 +2185,6 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) if (dep == erts_this_dist_entry) { Eterm id; - erts_deref_dist_entry(dep); if (IS_TRACED_FL(p, F_TRACE_SEND)) trace_send(p, to, msg); if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) @@ -2280,11 +2207,9 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) } ret = remote_send(p, dep, tp[1], to, msg, ctx); - if (ret != SEND_YIELD_CONTINUE) { - if (dep) { - erts_deref_dist_entry(dep); - } - } else { + if (ret == SEND_YIELD_CONTINUE) { + if (dep) + erts_ref_dist_entry(dep); ctx->dep_to_deref = dep; } return ret; @@ -2299,17 +2224,15 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) send_message: { ErtsProcLocks rp_locks = 0; Sint res; -#ifdef ERTS_SMP if (p == rp) rp_locks |= ERTS_PROC_LOCK_MAIN; -#endif /* send to local process */ res = erts_send_message(p, rp, &rp_locks, msg, 0); if (erts_use_sender_punish) res *= 4; else res = 0; - erts_smp_proc_unlock(rp, + erts_proc_unlock(rp, p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) : rp_locks); @@ -3021,17 +2944,17 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) { Eterm res; byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); - Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); - + Sint written; + int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS, + &written); if (i < 0) { erts_free(ERTS_ALC_T_TMP, (void *) buf); - i = erts_list_length(BIF_ARG_1); - if (i > MAX_ATOM_CHARACTERS) { + if (i == -2) { BIF_ERROR(BIF_P, SYSTEM_LIMIT); } BIF_ERROR(BIF_P, BADARG); } - res = erts_atom_put(buf, i, ERTS_ATOM_ENC_UTF8, 1); + res = erts_atom_put(buf, written, ERTS_ATOM_ENC_UTF8, 1); ASSERT(is_atom(res)); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(res); @@ -3042,8 +2965,9 @@ BIF_RETTYPE list_to_atom_1(BIF_ALIST_1) BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) { byte *buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, MAX_ATOM_SZ_LIMIT); - Sint i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS); - + Sint written; + int i = erts_unicode_list_to_buf(BIF_ARG_1, buf, MAX_ATOM_CHARACTERS, + &written); if (i < 0) { error: erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -3051,7 +2975,7 @@ BIF_RETTYPE list_to_existing_atom_1(BIF_ALIST_1) } else { Eterm a; - if (erts_atom_get((char *) buf, i, &a, ERTS_ATOM_ENC_UTF8)) { + if (erts_atom_get((char *) buf, written, &a, ERTS_ATOM_ENC_UTF8)) { erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(a); } else { @@ -3946,15 +3870,18 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) { Process* p = BIF_P; Eterm string = BIF_ARG_1; - Sint len = is_string(string); - char *str; + Sint len = erts_unicode_list_to_buf_len(string); + Sint written; + byte *str; + int res; - if (len <= 0) { + if (len < 0) { BIF_ERROR(p, BADARG); } - str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); - if (intlist_to_buf(string, str, len) != len) - erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); + str = (byte *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); + res = erts_unicode_list_to_buf(string, str, len, &written); + if (res != 0 || written != len) + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error (%d)\n", __FILE__, __LINE__, res); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3970,9 +3897,6 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) /**********************************************************************/ -#define HALT_MSG_SIZE 200 -static char halt_msg[HALT_MSG_SIZE+1]; - /* stop the system with exit code and flags */ BIF_RETTYPE halt_2(BIF_ALIST_2) { @@ -4012,29 +3936,30 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_halt_2], BIF_P, am_undefined, am_undefined); } else { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_exit(pos_int_code, ""); } } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_exit(ERTS_ABORT_EXIT, ""); } - else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { - Sint i; + else if (is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) { +# define HALT_MSG_SIZE 200 + static byte halt_msg[4*HALT_MSG_SIZE+1]; + Sint written; - if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE)) == -1) { + if (erts_unicode_list_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE, + &written) == -1 ) { goto error; } - if (i == -2) /* truncated string */ - i = HALT_MSG_SIZE; - ASSERT(i >= 0 && i <= HALT_MSG_SIZE); - halt_msg[i] = '\0'; + ASSERT(written >= 0 && written < sizeof(halt_msg)); + halt_msg[written] = '\0'; VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else @@ -4220,7 +4145,6 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) goto bad; if(dep == erts_this_dist_entry) { - erts_deref_dist_entry(dep); BIF_RET(make_internal_pid(make_pid_data(c, b))); } else { @@ -4240,13 +4164,10 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp->data.ui[0] = make_pid_data(c, b); MSO(BIF_P).first = (struct erl_off_heap_header*) etp; - erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } bad: - if (dep) - erts_deref_dist_entry(dep); if (buf) erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_ERROR(BIF_P, BADARG); @@ -4291,7 +4212,6 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) goto bad; if(dep == erts_this_dist_entry) { - erts_deref_dist_entry(dep); BIF_RET(make_internal_port(p)); } else { @@ -4311,13 +4231,10 @@ BIF_RETTYPE list_to_port_1(BIF_ALIST_1) etp->data.ui[0] = p; MSO(BIF_P).first = (struct erl_off_heap_header*) etp; - erts_deref_dist_entry(dep); BIF_RET(make_external_port(etp)); } bad: - if (dep) - erts_deref_dist_entry(dep); BIF_ERROR(BIF_P, BADARG); } @@ -4437,12 +4354,9 @@ BIF_RETTYPE list_to_ref_1(BIF_ALIST_1) res = make_external_ref(etp); } - erts_deref_dist_entry(dep); BIF_RET(res); bad: - if (dep) - erts_deref_dist_entry(dep); BIF_ERROR(BIF_P, BADARG); } @@ -4508,9 +4422,9 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) new_member->group_leader = BIF_ARG_1; else { locks &= ~ERTS_PROC_LOCK_STATUS; - erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); if (new_member == BIF_P - || !(erts_smp_atomic32_read_nob(&new_member->state) + || !(erts_atomic32_read_nob(&new_member->state) & ERTS_PSFLG_DIRTY_RUNNING)) { new_member->group_leader = STORE_NC_IN_PROC(new_member, BIF_ARG_1); @@ -4539,7 +4453,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) if (new_member == BIF_P) locks &= ~ERTS_PROC_LOCK_MAIN; if (locks) - erts_smp_proc_unlock(new_member, locks); + erts_proc_unlock(new_member, locks); if (await_x) { /* Wait for new_member to terminate; then badarg */ @@ -4566,9 +4480,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) if (BIF_ARG_1 == am_multi_scheduling) { if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) { -#ifndef ERTS_SMP - BIF_RET(am_disabled); -#else int block = (BIF_ARG_2 == am_block || BIF_ARG_2 == am_block_normal); int normal = (BIF_ARG_2 == am_block_normal @@ -4604,15 +4515,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); break; } -#endif } } else if (BIF_ARG_1 == am_schedulers_online) { -#ifndef ERTS_SMP - if (BIF_ARG_2 != make_small(1)) - goto error; - else - BIF_RET(make_small(1)); -#else Sint old_no; if (!is_small(BIF_ARG_2)) goto error; @@ -4636,7 +4540,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); break; } -#endif } else if (BIF_ARG_1 == am_fullsweep_after) { Uint16 nval; Uint oval; @@ -4644,7 +4547,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs, + oval = (Uint) erts_atomic32_xchg_nob(&erts_max_gen_gcs, (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { @@ -4654,13 +4557,13 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); H_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_bin_vheap_size) { @@ -4670,13 +4573,13 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_max_heap_size) { @@ -4694,14 +4597,14 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) hp = HAlloc(BIF_P, sz); old_value = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); H_MAX_SIZE = max_heap_size; H_MAX_FLAGS = max_heap_flags; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_value); } else if (BIF_ARG_1 == am_display_items) { @@ -4755,8 +4658,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } else if (BIF_ARG_1 == make_small(1)) { int i, max; ErtsMessage* mp; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); max = erts_ptab_max(&erts_proc); for (i = 0; i < max; i++) { @@ -4769,7 +4672,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) #endif p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + ERTS_MSGQ_MV_INQ2PRIVQ(p); mp = p->msg.first; while(mp != NULL) { #ifdef USE_VM_PROBES @@ -4782,14 +4685,14 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) } } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); } else if (BIF_ARG_1 == am_scheduler_wall_time) { if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; - erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time, + erts_aint32_t old = erts_atomic32_xchg_nob(&sched_wall_time, new); Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0); ASSERT(is_value(ref)); @@ -4798,7 +4701,6 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) ref, old ? am_true : am_false); } -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) } else if (BIF_ARG_1 == am_dirty_cpu_schedulers_online) { Sint old_no; if (!is_small(BIF_ARG_2)) @@ -4824,13 +4726,12 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); break; } -#endif } else if (BIF_ARG_1 == am_time_offset && ERTS_IS_ATOM_STR("finalize", BIF_ARG_2)) { ErtsTimeOffsetState res; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); res = erts_finalize_time_offset(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); switch (res) { case ERTS_TIME_OFFSET_PRELIMINARY: { DECL_AM(preliminary); @@ -4852,7 +4753,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) Eterm threads; if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { erts_aint32_t new = BIF_ARG_2 == am_true ? ERTS_MSACC_ENABLE : ERTS_MSACC_DISABLE; - erts_aint32_t old = erts_smp_atomic32_xchg_nob(&msacc, new); + erts_aint32_t old = erts_atomic32_xchg_nob(&msacc, new); Eterm ref = erts_msacc_request(BIF_P, new, &threads); if (is_non_value(ref)) BIF_RET(old ? am_true : am_false); @@ -4863,7 +4764,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) threads); } else if (BIF_ARG_2 == am_reset) { Eterm ref = erts_msacc_request(BIF_P, ERTS_MSACC_RESET, &threads); - erts_aint32_t old = erts_smp_atomic32_read_nob(&msacc); + erts_aint32_t old = erts_atomic32_read_nob(&msacc); ASSERT(is_value(ref)); BIF_TRAP3(await_msacc_mod_trap, BIF_P, @@ -4882,9 +4783,9 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) what = ERTS_SCHED_STAT_MODIFY_CLEAR; else goto error; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_sched_stat_modify(what); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("internal_cpu_topology", BIF_ARG_1)) { Eterm res = erts_set_cpu_topology(BIF_P, BIF_ARG_2); @@ -5022,7 +4923,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2) Eterm res = BIF_ARG_1; switch (BIF_ARG_2) { -#ifdef ERTS_SMP case am_multi_scheduling: { int msb = erts_is_multi_scheduling_blocked(); if (msb > 0) @@ -5033,7 +4933,6 @@ static BIF_RETTYPE bif_return_trap(BIF_ALIST_2) ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state"); break; } -#endif default: break; } @@ -5058,22 +4957,22 @@ static ERTS_INLINE int skip_current_msgq(Process *c_p) { int res; -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_CHECK) erts_proc_lc_chk_only_proc_main(c_p); #endif - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); if (ERTS_PROC_PENDING_EXIT(c_p)) { KILL_CATCHES(c_p); c_p->freason = EXC_EXIT; res = 0; } else { - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); c_p->msg.save = c_p->msg.last; res = 1; } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); return res; } @@ -5132,15 +5031,12 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, ep->info.mfa.module = m; ep->info.mfa.function = f; ep->info.mfa.arity = a; - ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[0] = BeamOpCodeAddr(op_apply_bif); ep->beam[1] = (BeamInstr) bif; } void erts_init_bif(void) { - erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); - /* * bif_return_trap/2 is a hidden BIF that bifs that need to * yield the calling process traps to. @@ -5179,8 +5075,8 @@ void erts_init_bif(void) await_msacc_mod_trap = erts_export_put(am_erts_internal, am_await_microstate_accounting_modifications, 3); - erts_smp_atomic32_init_nob(&sched_wall_time, 0); - erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); + erts_atomic32_init_nob(&sched_wall_time, 0); + erts_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } /* @@ -5198,15 +5094,14 @@ schedule(Process *c_p, Process *dirty_shadow_proc, Eterm module, Eterm function, int argc, Eterm *argv) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, - mfa, pc, (BeamInstr) em_apply_bif, + mfa, pc, BeamOpCodeAddr(op_apply_bif), dfunc, ifunc, module, function, argc, argv); } -#ifdef ERTS_DIRTY_SCHEDULERS static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) { @@ -5249,9 +5144,7 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) BIF_ERROR(BIF_P, freason); } -#endif /* ERTS_DIRTY_SCHEDULERS */ -extern BeamInstr* em_call_bif_e; static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I); BIF_RETTYPE @@ -5267,15 +5160,13 @@ erts_schedule_bif(Process *proc, Process *c_p, *dirty_shadow_proc; ErtsCodeMFA *mfa; -#ifdef ERTS_DIRTY_SCHEDULERS if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { dirty_shadow_proc = proc; c_p = proc->next; ASSERT(c_p->common.id == dirty_shadow_proc->common.id); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } else -#endif { dirty_shadow_proc = NULL; c_p = proc; @@ -5291,7 +5182,6 @@ erts_schedule_bif(Process *proc, * ibif - indirect bif */ -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t set, mask; mask = (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC); @@ -5314,11 +5204,7 @@ erts_schedule_bif(Process *proc, break; } - (void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set); -#else - dbif = call_bif; - ibif = bif; -#endif + (void) erts_atomic32_read_bset_nob(&c_p->state, mask, set); if (i == NULL) { ERTS_INTERNAL_ERROR("Missing instruction pointer"); @@ -5331,13 +5217,13 @@ erts_schedule_bif(Process *proc, mfa = &exp->info.mfa; } #endif - else if (em_call_bif_e == (BeamInstr *) *i) { + else if (BeamIsOpCode(*i, op_call_bif_e)) { /* Pointer to bif export in i+1 */ exp = (Export *) i[1]; pc = i; mfa = &exp->info.mfa; } - else if (em_apply_bif == (BeamInstr *) *i) { + else if (BeamIsOpCode(*i, op_apply_bif)) { /* Pointer to bif in i+1, and mfa in i-3 */ pc = c_p->cp; mfa = erts_code_to_codemfa(i); @@ -5359,7 +5245,7 @@ erts_schedule_bif(Process *proc, } if (dirty_shadow_proc) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); return THE_NON_VALUE; } @@ -5394,7 +5280,6 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I) return ret; } -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) @@ -5409,7 +5294,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * erts_aint32_t state; ASSERT(!c_p->scheduler_data); - state = erts_smp_atomic32_read_nob(&c_p->state); + state = erts_atomic32_read_nob(&c_p->state); ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); ASSERT(esdp); @@ -5423,7 +5308,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * bf = (ErtsBifFunc) I[1]; - erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p); @@ -5438,11 +5323,11 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * c_p_htop = c_p->htop; #endif - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); result = (*bf)(dirty_shadow_proc, reg, I); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); ASSERT(c_p_htop == c_p->htop); ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); @@ -5465,7 +5350,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * } else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { /* Dirty BIF did an ordinary trap... */ - ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + ASSERT(!(erts_atomic32_read_nob(&c_p->state) & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))); schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_trap, (void *) dirty_shadow_proc->i, @@ -5488,7 +5373,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * return exiting; } -#endif /* ERTS_DIRTY_SCHEDULERS */ #ifdef HARDDEBUG diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 01cca90a7a..a2bc883dbe 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -93,7 +93,7 @@ do { \ #define BUMP_REDS(p, gc) do { \ ASSERT(p); \ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\ + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\ (p)->fcalls -= (gc); \ if ((p)->fcalls < 0) { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) \ @@ -498,10 +498,8 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); -#endif BIF_RETTYPE erts_schedule_bif(Process *proc, diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index a8bbf5f8c1..f7b4451890 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -154,6 +154,12 @@ bif erlang:spawn_opt/1 bif erlang:setnode/2 bif erlang:setnode/3 bif erlang:dist_exit/3 +bif erlang:dist_get_stat/1 +bif erlang:dist_ctrl_input_handler/2 +bif erlang:dist_ctrl_put_data/2 +bif erlang:dist_ctrl_get_data/1 +bif erlang:dist_ctrl_get_data_notification/1 + # Static native functions in erts_internal bif erts_internal:port_info/1 @@ -437,7 +443,10 @@ bif erts_debug:dump_links/1 # # Lock counter bif's # -bif erts_debug:lock_counters/1 +bif erts_debug:lcnt_control/2 +bif erts_debug:lcnt_control/1 +bif erts_debug:lcnt_collect/0 +bif erts_debug:lcnt_clear/0 # # New Bifs in R8. @@ -676,3 +685,9 @@ bif math:ceil/1 bif math:fmod/2 bif os:set_signal/2 bif erts_internal:maps_to_list/2 + +# +# New in 20.1 +# + +bif erlang:iolist_to_iovec/1 diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab new file mode 100644 index 0000000000..0932b8b985 --- /dev/null +++ b/erts/emulator/beam/bif_instrs.tab @@ -0,0 +1,539 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +// ================================================================ +// All guards with zero arguments have special instructions, +// for example: +// +// self/0 +// node/0 +// +// All other guard BIFs take one or two arguments. +// ================================================================ + +CALL_GUARD_BIF(BF, TmpReg, Dst) { + Eterm result; + + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS; + PROCESS_MAIN_CHK_LOCKS(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); + result = (*$BF)(c_p, $TmpReg, I); + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + if (ERTS_LIKELY(is_value(result))) { + $Dst = result; + $NEXT0(); + } +} + +// Guard BIF in head. On failure, ignore the error and jump +// to the code for the next clause. We don't support tracing +// of guard BIFs. + +bif1(Fail, Bif, Src, Dst) { + ErtsBifFunc bf; + Eterm tmp_reg[1]; + + tmp_reg[0] = $Src; + bf = (BifFunction) $Bif; + $CALL_GUARD_BIF(bf, tmp_reg, $Dst); + + $FAIL($Fail); +} + +// +// Guard BIF in body. It can fail like any BIF. No trace support. +// + +bif1_body(Bif, Src, Dst) { + ErtsBifFunc bf; + Eterm tmp_reg[1]; + + tmp_reg[0] = $Src; + bf = (BifFunction) $Bif; + $CALL_GUARD_BIF(bf, tmp_reg, $Dst); + + reg[0] = tmp_reg[0]; + SWAPOUT; + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); + goto post_error_handling; +} + +// +// Guard bif in guard with two arguments ('and'/2, 'or'/2, 'xor'/2). +// + +i_bif2(Fail, Bif, Src1, Src2, Dst) { + Eterm tmp_reg[2]; + ErtsBifFunc bf; + + tmp_reg[0] = $Src1; + tmp_reg[1] = $Src2; + bf = (ErtsBifFunc) $Bif; + $CALL_GUARD_BIF(bf, tmp_reg, $Dst); + $FAIL($Fail); +} + +// +// Guard bif in body with two arguments ('and'/2, 'or'/2, 'xor'/2). +// + +i_bif2_body(Bif, Src1, Src2, Dst) { + Eterm tmp_reg[2]; + ErtsBifFunc bf; + + tmp_reg[0] = $Src1; + tmp_reg[1] = $Src2; + bf = (ErtsBifFunc) $Bif; + $CALL_GUARD_BIF(bf, tmp_reg, $Dst); + reg[0] = tmp_reg[0]; + reg[1] = tmp_reg[1]; + SWAPOUT; + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); + goto post_error_handling; +} + +// +// Garbage-collecting BIF with one argument in either guard or body. +// + +i_gc_bif1(Fail, Bif, Src, Live, Dst) { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) $Live; + + x(live) = $Src; + bf = (GcBifFunction) $Bif; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */ + $JUMP($Fail); + } + + /* Handle error in body. */ + x(0) = x(live); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); + goto post_error_handling; +} + +// +// Garbage-collecting BIF with two arguments in either guard or body. +// + +i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) $Live; + + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the second + * (i.e. 'live' should not be incremented below). + */ + x(live) = $Src1; + x(live+1) = $Src2; + live++; + + bf = (GcBifFunction) $Bif; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + + if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */ + $JUMP($Fail); + } + + /* Handle error in body. */ + live--; + x(0) = x(live); + x(1) = x(live+1); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); + goto post_error_handling; +} + +// +// Garbage-collecting BIF with three arguments in either guard or body. +// + +i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) $Live; + + /* + * XXX This calling convention does not make sense. 'live' + * should point out the first argument, not the third + * (i.e. 'live' should not be incremented below). + */ + x(live) = x(SCRATCH_X_REG); + x(live+1) = $Src2; + x(live+2) = $Src3; + live += 2; + + bf = (GcBifFunction) $Bif; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + if (ERTS_LIKELY(is_value(result))) { + $REFRESH_GEN_DEST(); + $Dst = result; + $NEXT0(); + } + + /* Handle error in guard. */ + if (ERTS_LIKELY($Fail != 0)) { + $JUMP($Fail); + } + + /* Handle error in body. */ + live -= 2; + x(0) = x(live); + x(1) = x(live+1); + x(2) = x(live+2); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); + goto post_error_handling; +} + +// +// The most general BIF call. The BIF may build any amount of data +// on the heap. The result is always returned in r(0). +// +call_bif(Exp) { + ErtsBifFunc bf; + Eterm result; + ErlHeapFragment *live_hf_end; + Export *export = (Export*) $Exp; + + if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; + goto context_switch3; + } + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), + GET_BIF_ADDRESS(export)); + + bf = GET_BIF_ADDRESS(export); + + PRE_BIF_SWAPOUT(c_p); + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS - 1; + if (FCALLS <= 0) { + save_calls(c_p, export); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_HOLE_CHECK(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + if (ERTS_IS_GC_DESIRED(c_p)) { + Uint arity = GET_BIF_ARITY(export); + result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, + reg, arity); + E = c_p->stop; + } + PROCESS_MAIN_CHK_LOCKS(c_p); + HTOP = HEAP_TOP(c_p); + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { + ERTS_MSACC_UPDATE_CACHE_X(); + } + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (ERTS_LIKELY(is_value(result))) { + r(0) = result; + CHECK_TERM(r(0)); + $NEXT0(); + } else if (c_p->freason == TRAP) { + SET_CP(c_p, I+2); + SET_I(c_p->i); + SWAPIN; + Dispatch(); + } + + /* + * Error handling. SWAPOUT is not needed because it was done above. + */ + ASSERT(c_p->stop == E); + I = handle_error(c_p, I, reg, &export->info.mfa); + goto post_error_handling; +} + +// +// Send is almost a standard call-BIF with two arguments, except for: +// 1. It cannot be traced. +// 2. There is no pointer to the send_2 function stored in +// the instruction. +// + +send() { + Eterm result; + + if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = 2; + c_p->current = NULL; + goto context_switch3; + } + + PRE_BIF_SWAPOUT(c_p); + c_p->fcalls = FCALLS - 1; + result = erl_send(c_p, r(0), x(1)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + HTOP = HEAP_TOP(c_p); + FCALLS = c_p->fcalls; + if (ERTS_LIKELY(is_value(result))) { + r(0) = result; + CHECK_TERM(r(0)); + } else if (c_p->freason == TRAP) { + SET_CP(c_p, I+1); + SET_I(c_p->i); + SWAPIN; + Dispatch(); + } else { + goto find_func_info; + } +} + +call_nif := nif_bif.call_nif.epilogue; +apply_bif := nif_bif.apply_bif.epilogue; + +nif_bif.head() { + Eterm nif_bif_result; + Eterm bif_nif_arity; + BifFunction vbf; + ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; +} + +nif_bif.call_nif() { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct + */ + + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + + HEAVY_SWAPOUT; + + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = codemfa->arity; + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + { + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + ASSERT(c_p->scheduler_data); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + if (env.exception_thrown) + nif_bif_result = THE_NON_VALUE; + erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); + + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + ASSERT(!env.exiting); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + + DTRACE_NIF_RETURN(c_p, codemfa); +} + +nif_bif.apply_bif() { + /* + * At this point, I points to the code[0] in the export entry for + * the BIF: + * + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function + */ + + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + goto context_switch; + } + + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); + + + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; + $SET_CP_I_ABS(I); /* In case we apply check_process_code/2. */ + c_p->arity = 0; /* To allow garbage collection on ourselves + * (check_process_code/2). + */ + DTRACE_BIF_ENTRY(c_p, codemfa); + + SWAPOUT; + ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); + c_p->fcalls = FCALLS - 1; + vbf = (BifFunction) Arg(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = codemfa->arity; + ASSERT(bif_nif_arity <= 4); + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + { + ErtsBifFunc bf = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); + nif_bif_result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + /* We have to update the cache if we are enabled in order + to make sure no book keeping is done after we disabled + msacc. We don't always do this as it is quite expensive. */ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) + ERTS_MSACC_UPDATE_CACHE_X(); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + DTRACE_BIF_RETURN(c_p, codemfa); +} + +nif_bif.epilogue() { + ERTS_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + if (ERTS_IS_GC_DESIRED(c_p)) { + nif_bif_result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, + nif_bif_result, + reg, bif_nif_arity); + } + SWAPIN; /* There might have been a garbage collection. */ + FCALLS = c_p->fcalls; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + if (ERTS_LIKELY(is_value(nif_bif_result))) { + r(0) = nif_bif_result; + CHECK_TERM(r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); + } else if (c_p->freason == TRAP) { + SET_I(c_p->i); + if (c_p->flags & F_HIBERNATE_SCHED) { + c_p->flags &= ~F_HIBERNATE_SCHED; + goto do_schedule; + } + Dispatch(); + } + I = handle_error(c_p, c_p->cp, reg, c_p->current); + goto post_error_handling; +} diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 7128b8ed23..5eaf262cd8 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1293,8 +1293,11 @@ static dsize_t I_bxor(ErtsDigit* x, dsize_t xl, short xsgn, *r++ = ~c ^ *y++; x++; } - while(xl--) - *r++ = ~*x++; + while(xl--) { + DSUBb(*x,0,b,c); + *r++ = ~c; + x++; + } } else { ErtsDigit b1, b2; @@ -1312,7 +1315,9 @@ static dsize_t I_bxor(ErtsDigit* x, dsize_t xl, short xsgn, x++; y++; } while(xl--) { - *r++ = *x++; + DSUBb(*x,0,b1,c1); + *r++ = c1; + x++; } } } diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 48efce20e7..7556205063 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -70,7 +70,20 @@ typedef Uint dsize_t; /* Vector size type */ /* Check for small */ #define IS_USMALL(sgn,x) ((sgn) ? ((x) <= MAX_SMALL+1) : ((x) <= MAX_SMALL)) -#define IS_SSMALL(x) (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL)) + +/* + * It seems that both clang and gcc will generate sub-optimal code + * for the more obvious way to write the range check: + * + * #define IS_SSMALL(x) (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL)) + * + * Note that IS_SSMALL() may be used in the 32-bit emulator with + * a Uint64 argument. Therefore, we must test the size of the argument + * to ensure that the cast does not discard the high-order 32 bits. + */ +#define _IS_SSMALL32(x) (((Uint32) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) +#define _IS_SSMALL64(x) (((Uint64) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) +#define IS_SSMALL(x) (sizeof(x) == sizeof(Uint32) ? _IS_SSMALL32(x) : _IS_SSMALL64(x)) /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 76a0c5c716..35b2365655 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -109,8 +109,8 @@ process_killer(void) ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_aint32_t state; erts_proc_inc_refc(rp); - erts_smp_proc_lock(rp, rp_locks); - state = erts_smp_atomic32_read_acqb(&rp->state); + erts_proc_lock(rp, rp_locks); + state = erts_atomic32_read_acqb(&rp->state); if (state & (ERTS_PSFLG_FREE | ERTS_PSFLG_EXITING | ERTS_PSFLG_ACTIVE @@ -132,7 +132,7 @@ process_killer(void) NULL, 0); } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); } case 'n': br = 1; break; @@ -219,7 +219,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) /* Display the state */ erts_print(to, to_arg, "State: "); - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); erts_dump_process_state(to, to_arg, state); if (state & ERTS_PSFLG_GC) { garbing = 1; @@ -258,7 +258,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Spawned by: %T\n", p->parent); approx_started = (time_t) p->approx_started; erts_print(to, to_arg, "Started: %s", ctime(&approx_started)); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + ERTS_MSGQ_MV_INQ2PRIVQ(p); erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len); /* display the message queue only if there is anything in it */ @@ -344,9 +344,7 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_program_counter_info(to, to_arg, p); } else { erts_print(to, to_arg, "Stack dump:\n"); -#ifdef ERTS_SMP if (!garbing) -#endif erts_stack_dump(to, to_arg, p); } @@ -358,11 +356,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) static void print_garb_info(fmtfn_t to, void *to_arg, Process* p) { -#ifdef ERTS_SMP /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ if (!ERTS_IS_CRASH_DUMPING) return; -#endif erts_print(to, to_arg, "New heap start: %bpX\n", p->heap); erts_print(to, to_arg, "New heap top: %bpX\n", p->htop); erts_print(to, to_arg, "Stack top: %bpX\n", p->stop); @@ -512,7 +508,7 @@ do_break(void) erts_free_read_env(mode); #endif /* __WIN32__ */ - ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(erts_thr_progress_is_blocking()); erts_printf("\n" "BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded\n" @@ -698,9 +694,7 @@ static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len) void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) { -#ifdef ERTS_SMP ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */ -#endif int fd; size_t envsz; time_t now; @@ -717,7 +711,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; -#ifdef ERTS_SMP /* Order all managed threads to block, this has to be done first to guarantee that this is the only thread to generate crash dump. */ @@ -741,12 +734,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #endif /* Allow us to pass certain places without locking... */ - erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); - erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1); + erts_atomic32_set_mb(&erts_writing_erl_crash_dump, 1); + erts_tsd_set(erts_is_crash_dumping_key, (void *) 1); -#else /* !ERTS_SMP */ - erts_writing_erl_crash_dump = 1; -#endif /* ERTS_SMP */ envsz = sizeof(env); /* ERL_CRASH_DUMP_SECONDS not set @@ -841,7 +831,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_print_nif_taints(to, to_arg); erts_cbprintf(to, to_arg, "Atoms: %d\n", atom_table_size()); -#ifdef USE_THREADS /* We want to note which thread it was that called erts_exit */ if (erts_get_scheduler_data()) { erts_cbprintf(to, to_arg, "Calling Thread: scheduler:%d\n", @@ -852,9 +841,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) else erts_cbprintf(to, to_arg, "Calling Thread: %p\n", erts_thr_self()); } -#else - erts_cbprintf(to, to_arg, "Calling Thread: scheduler:1\n"); -#endif #if defined(ERTS_HAVE_TRY_CATCH) @@ -873,7 +859,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } #endif -#ifdef ERTS_SMP #ifdef ERTS_SYS_SUSPEND_SIGNAL @@ -895,7 +880,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) */ erts_thr_progress_fatal_error_wait(60000); /* Either worked or not... */ -#endif #ifndef ERTS_HAVE_TRY_CATCH /* This is safe to call here, as all schedulers are blocked */ diff --git a/erts/emulator/beam/bs_instrs.tab b/erts/emulator/beam/bs_instrs.tab new file mode 100644 index 0000000000..9f03b19731 --- /dev/null +++ b/erts/emulator/beam/bs_instrs.tab @@ -0,0 +1,1021 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +%if ARCH_64 +BS_SAFE_MUL(A, B, Fail, Dst) { + Uint64 res = ($A) * ($B); + if (res / $B != $A) { + $Fail; + } + $Dst = res; +} +%else +BS_SAFE_MUL(A, B, Fail, Dst) { + Uint64 res = (Uint64)($A) * (Uint64)($B); + if ((res >> (8*sizeof(Uint))) != 0) { + $Fail; + } + $Dst = res; +} +%endif + +BS_GET_FIELD_SIZE(Bits, Unit, Fail, Dst) { + Sint signed_size; + Uint uint_size; + Uint temp_bits; + + if (is_small($Bits)) { + signed_size = signed_val($Bits); + if (signed_size < 0) { + $Fail; + } + uint_size = (Uint) signed_size; + } else { + if (!term_to_Uint($Bits, &temp_bits)) { + $Fail; + } + uint_size = temp_bits; + } + $BS_SAFE_MUL(uint_size, $Unit, $Fail, $Dst); +} + +BS_GET_UNCHECKED_FIELD_SIZE(Bits, Unit, Fail, Dst) { + Sint signed_size; + Uint uint_size; + Uint temp_bits; + + if (is_small($Bits)) { + signed_size = signed_val($Bits); + if (signed_size < 0) { + $Fail; + } + uint_size = (Uint) signed_size; + } else { + if (!term_to_Uint($Bits, &temp_bits)) { + $Fail; + } + uint_size = temp_bits; + } + $Dst = uint_size * $Unit; +} + +TEST_BIN_VHEAP(VNh, Nh, Live) { + Uint need = $Nh; + if (E - HTOP < need || MSO(c_p).overhead + $VNh >= BIN_VHEAP_SZ(c_p)) { + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + } + HEAP_SPACE_VERIFIED(need); +} + +i_bs_get_binary_all2(Fail, Ms, Live, Unit, Dst) { + ErlBinMatchBuffer *_mb; + Eterm _result; + + $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live); + _mb = ms_matchbuffer($Ms); + if (((_mb->size - _mb->offset) % $Unit) == 0) { + LIGHT_SWAPOUT; + _result = erts_bs_get_binary_all_2(c_p, _mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + ASSERT(is_value(_result)); + $Dst = _result; + } else { + HEAP_SPACE_VERIFIED(0); + $FAIL($Fail); + } +} + +i_bs_get_binary2(Fail, Ms, Live, Sz, Flags, Dst) { + ErlBinMatchBuffer *_mb; + Eterm _result; + Uint _size; + $BS_GET_FIELD_SIZE($Sz, (($Flags) >> 3), $FAIL($Fail), _size); + $GC_TEST(0, ERL_SUB_BIN_SIZE, $Live); + _mb = ms_matchbuffer($Ms); + LIGHT_SWAPOUT; + _result = erts_bs_get_binary_2(c_p, _size, $Flags, _mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(_result)) { + $FAIL($Fail); + } else { + $Dst = _result; + } +} + +i_bs_get_binary_imm2(Fail, Ms, Live, Sz, Flags, Dst) { + ErlBinMatchBuffer *_mb; + Eterm _result; + $GC_TEST(0, heap_bin_size(ERL_ONHEAP_BIN_LIMIT), $Live); + _mb = ms_matchbuffer($Ms); + LIGHT_SWAPOUT; + _result = erts_bs_get_binary_2(c_p, $Sz, $Flags, _mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(_result)) { + $FAIL($Fail); + } else { + $Dst = _result; + } +} + +i_bs_get_float2(Fail, Ms, Live, Sz, Flags, Dst) { + ErlBinMatchBuffer *_mb; + Eterm _result; + Sint _size; + + if (!is_small($Sz) || (_size = unsigned_val($Sz)) > 64) { + $FAIL($Fail); + } + _size *= (($Flags) >> 3); + $GC_TEST(0, FLOAT_SIZE_OBJECT, $Live); + _mb = ms_matchbuffer($Ms); + LIGHT_SWAPOUT; + _result = erts_bs_get_float_2(c_p, _size, ($Flags), _mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(_result)) { + $FAIL($Fail); + } else { + $Dst = _result; + } +} + +i_bs_skip_bits2(Fail, Ms, Bits, Unit) { + ErlBinMatchBuffer *_mb; + size_t new_offset; + Uint _size; + + _mb = ms_matchbuffer($Ms); + $BS_GET_FIELD_SIZE($Bits, $Unit, $FAIL($Fail), _size); + new_offset = _mb->offset + _size; + if (new_offset <= _mb->size) { + _mb->offset = new_offset; + } else { + $FAIL($Fail); + } +} + +i_bs_skip_bits_all2(Fail, Ms, Unit) { + ErlBinMatchBuffer *_mb; + _mb = ms_matchbuffer($Ms); + if (((_mb->size - _mb->offset) % $Unit) == 0) { + _mb->offset = _mb->size; + } else { + $FAIL($Fail); + } +} + +i_bs_skip_bits_imm2(Fail, Ms, Bits) { + ErlBinMatchBuffer *_mb; + size_t new_offset; + _mb = ms_matchbuffer($Ms); + new_offset = _mb->offset + ($Bits); + if (new_offset <= _mb->size) { + _mb->offset = new_offset; + } else { + $FAIL($Fail); + } +} + +i_new_bs_put_binary(Fail, Sz, Flags, Src) { + Sint _size; + $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); + if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), _size))) { + $BADARG($Fail); + } +} + +i_new_bs_put_binary_all(Fail, Src, Unit) { + if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2(($Src), ($Unit)))) { + $BADARG($Fail); + } +} + +i_new_bs_put_binary_imm(Fail, Sz, Src) { + if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2(($Src), ($Sz)))) { + $BADARG($Fail); + } +} + +i_new_bs_put_float(Fail, Sz, Flags, Src) { + Sint _size; + $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); + if (!erts_new_bs_put_float(c_p, ($Src), _size, ($Flags))) { + $BADARG($Fail); + } +} + +i_new_bs_put_float_imm(Fail, Sz, Flags, Src) { + if (!erts_new_bs_put_float(c_p, ($Src), ($Sz), ($Flags))) { + $BADARG($Fail); + } +} + +i_new_bs_put_integer(Fail, Sz, Flags, Src) { + Sint _size; + $BS_GET_UNCHECKED_FIELD_SIZE($Sz, (($Flags) >> 3), $BADARG($Fail), _size); + if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), _size, ($Flags)))) { + $BADARG($Fail); + } +} + +i_new_bs_put_integer_imm(Fail, Sz, Flags, Src) { + if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3(($Src), ($Sz), ($Flags)))) { + $BADARG($Fail); + } +} + +# +# i_bs_init* +# + +i_bs_init_fail_heap := bs_init.fail_heap.verify.execute; +i_bs_init_fail := bs_init.fail.verify.execute; +i_bs_init := bs_init.plain.execute; +i_bs_init_heap := bs_init.heap.execute; + +bs_init.head() { + Eterm BsOp1; + Eterm BsOp2; +} + +bs_init.fail_heap(Size, HeapAlloc) { + BsOp1 = $Size; + BsOp2 = $HeapAlloc; +} + +bs_init.fail(Size) { + BsOp1 = $Size; + BsOp2 = 0; +} + +bs_init.plain(Size) { + BsOp1 = $Size; + BsOp2 = 0; +} + +bs_init.heap(Size, HeapAlloc) { + BsOp1 = $Size; + BsOp2 = $HeapAlloc; +} + +bs_init.verify(Fail) { + if (is_small(BsOp1)) { + Sint size = signed_val(BsOp1); + if (size < 0) { + $BADARG($Fail); + } + BsOp1 = (Eterm) size; + } else { + Uint bytes; + + if (!term_to_Uint(BsOp1, &bytes)) { + c_p->freason = bytes; + $FAIL_HEAD_OR_BODY($Fail); + } + if ((bytes >> (8*sizeof(Uint)-3)) != 0) { + $SYSTEM_LIMIT($Fail); + } + BsOp1 = (Eterm) bytes; + } +} + +bs_init.execute(Live, Dst) { + if (BsOp1 <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + Uint bin_need; + + bin_need = heap_bin_size(BsOp1); + erts_bin_offset = 0; + erts_writable_bin = 0; + $GC_TEST(0, bin_need+BsOp2+ERL_SUB_BIN_SIZE, $Live); + hb = (ErlHeapBin *) HTOP; + HTOP += bin_need; + hb->thing_word = header_heap_bin(BsOp1); + hb->size = BsOp1; + erts_current_bin = (byte *) hb->data; + $Dst = make_binary(hb); + } else { + Binary* bptr; + ProcBin* pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + $TEST_BIN_VHEAP(BsOp1 / sizeof(Eterm), + BsOp2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, $Live); + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(BsOp1); + erts_current_bin = (byte *) bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *) HTOP; + HTOP += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = BsOp1; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; + pb->val = bptr; + pb->bytes = (byte*) bptr->orig_bytes; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(c_p)), BsOp1 / sizeof(Eterm)); + + $Dst = make_binary(pb); + } +} + +# +# i_bs_init_bits* +# + +i_bs_init_bits := bs_init_bits.plain.execute; +i_bs_init_bits_heap := bs_init_bits.heap.execute; +i_bs_init_bits_fail := bs_init_bits.fail.verify.execute; +i_bs_init_bits_fail_heap := bs_init_bits.fail_heap.verify.execute; + +bs_init_bits.head() { + Eterm num_bits_term; + Uint num_bits; + Uint alloc; +} + +bs_init_bits.plain(NumBits) { + num_bits = $NumBits; + alloc = 0; +} + +bs_init_bits.heap(NumBits, Alloc) { + num_bits = $NumBits; + alloc = $Alloc; +} + +bs_init_bits.fail(NumBitsTerm) { + num_bits_term = $NumBitsTerm; + alloc = 0; +} + +bs_init_bits.fail_heap(NumBitsTerm, Alloc) { + num_bits_term = $NumBitsTerm; + alloc = $Alloc; +} + +bs_init_bits.verify(Fail) { + if (is_small(num_bits_term)) { + Sint size = signed_val(num_bits_term); + if (size < 0) { + $BADARG($Fail); + } + num_bits = (Uint) size; + } else { + Uint bits; + + if (!term_to_Uint(num_bits_term, &bits)) { + c_p->freason = bits; + $FAIL_HEAD_OR_BODY($Fail); + } + num_bits = (Uint) bits; + } +} + +bs_init_bits.execute(Live, Dst) { + Eterm new_binary; + Uint num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3; + + if (num_bits & 7) { + alloc += ERL_SUB_BIN_SIZE; + } + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + alloc += heap_bin_size(num_bytes); + } else { + alloc += PROC_BIN_SIZE; + } + $test_heap(alloc, $Live); + + /* num_bits = Number of bits to build + * num_bytes = Number of bytes to allocate in the binary + * alloc = Total number of words to allocate on heap + * Operands: NotUsed NotUsed Dst + */ + if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) { + ErlHeapBin* hb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + hb = (ErlHeapBin *) HTOP; + HTOP += heap_bin_size(num_bytes); + hb->thing_word = header_heap_bin(num_bytes); + hb->size = num_bytes; + erts_current_bin = (byte *) hb->data; + new_binary = make_binary(hb); + + do_bits_sub_bin: + if (num_bits & 7) { + ErlSubBin* sb; + + sb = (ErlSubBin *) HTOP; + HTOP += ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = num_bytes - 1; + sb->bitsize = num_bits & 7; + sb->offs = 0; + sb->bitoffs = 0; + sb->is_writable = 0; + sb->orig = new_binary; + new_binary = make_binary(sb); + } + HEAP_SPACE_VERIFIED(0); + $Dst = new_binary; + } else { + Binary* bptr; + ProcBin* pb; + + erts_bin_offset = 0; + erts_writable_bin = 0; + + /* + * Allocate the binary struct itself. + */ + bptr = erts_bin_nrml_alloc(num_bytes); + erts_current_bin = (byte *) bptr->orig_bytes; + + /* + * Now allocate the ProcBin on the heap. + */ + pb = (ProcBin *) HTOP; + HTOP += PROC_BIN_SIZE; + pb->thing_word = HEADER_PROC_BIN; + pb->size = num_bytes; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; + pb->val = bptr; + pb->bytes = (byte*) bptr->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); + new_binary = make_binary(pb); + goto do_bits_sub_bin; + } +} + +bs_add(Fail, Src1, Src2, Unit, Dst) { + Eterm Op1 = $Src1; + Eterm Op2 = $Src2; + Uint unit = $Unit; + + if (is_both_small(Op1, Op2)) { + Sint Arg1 = signed_val(Op1); + Sint Arg2 = signed_val(Op2); + + if (Arg1 >= 0 && Arg2 >= 0) { + $BS_SAFE_MUL(Arg2, unit, $SYSTEM_LIMIT($Fail), Op1); + Op1 += Arg1; + + store_bs_add_result: + if (Op1 <= MAX_SMALL) { + Op1 = make_small(Op1); + } else { + /* + * May generate a heap fragment, but in this + * particular case it is OK, since the value will be + * stored into an x register (the GC will scan x + * registers for references to heap fragments) and + * there is no risk that value can be stored into a + * location that is not scanned for heap-fragment + * references (such as the heap). + */ + SWAPOUT; + Op1 = erts_make_integer(Op1, c_p); + HTOP = HEAP_TOP(c_p); + } + $Dst = Op1; + $NEXT0(); + } + $BADARG($Fail); + } else { + Uint a; + Uint b; + Uint c; + + /* + * Now we know that one of the arguments is + * not a small. We must convert both arguments + * to Uints and check for errors at the same time. + * + * Error checking is tricky. + * + * If one of the arguments is not numeric or + * not positive, the error reason is BADARG. + * + * Otherwise if both arguments are numeric, + * but at least one argument does not fit in + * an Uint, the reason is SYSTEM_LIMIT. + */ + + if (!term_to_Uint(Op1, &a)) { + if (a == BADARG) { + $BADARG($Fail); + } + if (!term_to_Uint(Op2, &b)) { + c_p->freason = b; + $FAIL_HEAD_OR_BODY($Fail); + } + $SYSTEM_LIMIT($Fail); + } else if (!term_to_Uint(Op2, &b)) { + c_p->freason = b; + $FAIL_HEAD_OR_BODY($Fail); + } + + /* + * The arguments are now correct and stored in a and b. + */ + + $BS_SAFE_MUL(b, unit, $SYSTEM_LIMIT($Fail), c); + Op1 = a + c; + if (Op1 < a) { + /* + * If the result is less than one of the + * arguments, there must have been an overflow. + */ + $SYSTEM_LIMIT($Fail); + } + goto store_bs_add_result; + } + /* No fallthrough */ + ASSERT(0); +} + +bs_put_string(Len, Ptr) { + erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) $Ptr, $Len)); +} + +i_bs_append(Fail, ExtraHeap, Live, Unit, Size, Dst) { + Uint live = $Live; + Uint res; + + HEAVY_SWAPOUT; + reg[live] = x(SCRATCH_X_REG); + res = erts_bs_append(c_p, reg, live, $Size, $ExtraHeap, $Unit); + HEAVY_SWAPIN; + if (is_non_value(res)) { + /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */ + $FAIL_HEAD_OR_BODY($Fail); + } + $Dst = res; +} + +i_bs_private_append(Fail, Unit, Size, Src, Dst) { + Eterm res; + + res = erts_bs_private_append(c_p, $Src, $Size, $Unit); + if (is_non_value(res)) { + /* c_p->freason is already set (to BADARG or SYSTEM_LIMIT). */ + $FAIL_HEAD_OR_BODY($Fail); + } + $Dst = res; +} + +bs_init_writable() { + HEAVY_SWAPOUT; + r(0) = erts_bs_init_writable(c_p, r(0)); + HEAVY_SWAPIN; +} + +i_bs_utf8_size(Src, Dst) { + Eterm arg = $Src; + Eterm result; + + /* + * Calculate the number of bytes needed to encode the source + * operand to UTF-8. If the source operand is invalid (e.g. wrong + * type or range) we return a nonsense integer result (0 or 4). We + * can get away with that because we KNOW that bs_put_utf8 will do + * full error checking. + */ + + if (arg < make_small(0x80UL)) { + result = make_small(1); + } else if (arg < make_small(0x800UL)) { + result = make_small(2); + } else if (arg < make_small(0x10000UL)) { + result = make_small(3); + } else { + result = make_small(4); + } + $Dst = result; +} + +i_bs_put_utf8(Fail, Src) { + if (!erts_bs_put_utf8(ERL_BITS_ARGS_1($Src))) { + $BADARG($Fail); + } +} + +i_bs_utf16_size(Src, Dst) { + Eterm arg = $Src; + Eterm result = make_small(2); + + /* + * Calculate the number of bytes needed to encode the source + * operarand to UTF-16. If the source operand is invalid (e.g. wrong + * type or range) we return a nonsense integer result (2 or 4). We + * can get away with that because we KNOW that bs_put_utf16 will do + * full error checking. + */ + + if (arg >= make_small(0x10000UL)) { + result = make_small(4); + } + $Dst = result; +} + +bs_put_utf16(Fail, Flags, Src) { + if (!erts_bs_put_utf16(ERL_BITS_ARGS_2($Src, $Flags))) { + $BADARG($Fail); + } +} + +// Validate a value about to be stored in a binary. +i_bs_validate_unicode(Fail, Src) { + Eterm val = $Src; + + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (if the term is a bignum, it could + * slip through the test, and there is no further test that + * would catch it, since bit syntax construction silently masks + * too big numbers). + */ + if (is_not_small(val) || val > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) { + $BADARG($Fail); + } +} + +// Validate a value that has been matched out. +i_bs_validate_unicode_retract(Fail, Src, Ms) { + /* + * There is no need to untag the integer, but it IS necessary + * to make sure it is small (a bignum pointer could fall in + * the valid range). + */ + + Eterm i = $Src; + if (is_not_small(i) || i > make_small(0x10FFFFUL) || + (make_small(0xD800UL) <= i && i <= make_small(0xDFFFUL))) { + Eterm ms = $Ms; /* Match context */ + ErlBinMatchBuffer* mb; + + /* Invalid value. Retract the position in the binary. */ + mb = ms_matchbuffer(ms); + mb->offset -= 32; + $BADARG($Fail); + } +} + + +// +// Matching of binaries. +// + +i_bs_start_match2 := bs_start_match.fetch.execute; + +bs_start_match.head() { + Eterm context; +} + +bs_start_match.fetch(Src) { + context = $Src; +} + +bs_start_match.execute(Fail, Live, Slots, Dst) { + Uint slots; + Uint live; + Eterm header; + if (!is_boxed(context)) { + $FAIL($Fail); + } + header = *boxed_val(context); + slots = $Slots; + live = $Live; + if (header_is_bin_matchstate(header)) { + ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); + Uint actual_slots = HEADER_NUM_SLOTS(header); + ms->save_offset[0] = ms->mb.offset; + if (actual_slots < slots) { + ErlBinMatchState* dst; + Uint live = $Live; + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + + $GC_TEST_PRESERVE(wordsneeded, live, context); + ms = (ErlBinMatchState *) boxed_val(context); + dst = (ErlBinMatchState *) HTOP; + *dst = *ms; + *HTOP = HEADER_BIN_MATCHSTATE(slots); + HTOP += wordsneeded; + HEAP_SPACE_VERIFIED(0); + $Dst = make_matchstate(dst); + } + } else if (is_binary_header(header)) { + Eterm result; + Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); + $GC_TEST_PRESERVE(wordsneeded, live, context); + HEAP_TOP(c_p) = HTOP; +#ifdef DEBUG + c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ +#endif + result = erts_bs_start_match_2(c_p, context, slots); + HTOP = HEAP_TOP(c_p); + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + $FAIL($Fail); + } + $Dst = result; + } else { + $FAIL($Fail); + } +} + +bs_test_zero_tail2(Fail, Ctx) { + ErlBinMatchBuffer *_mb; + _mb = (ErlBinMatchBuffer*) ms_matchbuffer($Ctx); + if (_mb->size != _mb->offset) { + $FAIL($Fail); + } +} + +bs_test_tail_imm2(Fail, Ctx, Offset) { + ErlBinMatchBuffer *_mb; + _mb = ms_matchbuffer($Ctx); + if (_mb->size - _mb->offset != $Offset) { + $FAIL($Fail); + } +} + +bs_test_unit(Fail, Ctx, Unit) { + ErlBinMatchBuffer *_mb; + _mb = ms_matchbuffer($Ctx); + if ((_mb->size - _mb->offset) % $Unit) { + $FAIL($Fail); + } +} + +bs_test_unit8(Fail, Ctx) { + ErlBinMatchBuffer *_mb; + _mb = ms_matchbuffer($Ctx); + if ((_mb->size - _mb->offset) & 7) { + $FAIL($Fail); + } +} + +i_bs_get_integer_8(Ctx, Fail, Dst) { + Eterm _result; + ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + + if (_mb->size - _mb->offset < 8) { + $FAIL($Fail); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); + } else { + _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); + _mb->offset += 8; + } + $Dst = _result; +} + +i_bs_get_integer_16(Ctx, Fail, Dst) { + Eterm _result; + ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + + if (_mb->size - _mb->offset < 16) { + $FAIL($Fail); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); + } else { + _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); + _mb->offset += 16; + } + $Dst = _result; +} + +%if ARCH_64 +i_bs_get_integer_32(Ctx, Fail, Dst) { + Uint32 _integer; + ErlBinMatchBuffer* _mb = ms_matchbuffer($Ctx); + + if (_mb->size - _mb->offset < 32) { + $FAIL($Fail); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _integer = erts_bs_get_unaligned_uint32(_mb); + } else { + _integer = get_int32(_mb->base + _mb->offset/8); + } + _mb->offset += 32; + $Dst = make_small(_integer); +} +%endif + +i_bs_get_integer_imm := bs_get_integer.fetch.execute; +i_bs_get_integer_small_imm := bs_get_integer.fetch_small.execute; + +bs_get_integer.head() { + Eterm Ms, Sz; +} + +bs_get_integer.fetch(Ctx, Size, Live) { + Uint wordsneeded; + Ms = $Ctx; + Sz = $Size; + wordsneeded = 1+WSIZE(NBYTES(Sz)); + $GC_TEST_PRESERVE(wordsneeded, $Live, Ms); +} + +bs_get_integer.fetch_small(Ctx, Size) { + Ms = $Ctx; + Sz = $Size; +} + +bs_get_integer.execute(Fail, Flags, Dst) { + ErlBinMatchBuffer* mb; + Eterm result; + + mb = ms_matchbuffer(Ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, Sz, $Flags, mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + $FAIL($Fail); + } + $Dst = result; +} + +i_bs_get_integer(Fail, Live, FlagsAndUnit, Ms, Sz, Dst) { + Uint flags; + Uint size; + Eterm ms; + ErlBinMatchBuffer* mb; + Eterm result; + + flags = $FlagsAndUnit; + ms = $Ms; + $BS_GET_FIELD_SIZE($Sz, (flags >> 3), $FAIL($Fail), size); + if (size >= SMALL_BITS) { + Uint wordsneeded; + /* Check bits size before potential gc. + * We do not want a gc and then realize we don't need + * the allocated space (i.e. if the op fails). + * + * Remember to re-acquire the matchbuffer after gc. + */ + + mb = ms_matchbuffer(ms); + if (mb->size - mb->offset < size) { + $FAIL($Fail); + } + wordsneeded = 1+WSIZE(NBYTES((Uint) size)); + $GC_TEST_PRESERVE(wordsneeded, $Live, ms); + } + mb = ms_matchbuffer(ms); + LIGHT_SWAPOUT; + result = erts_bs_get_integer_2(c_p, size, flags, mb); + LIGHT_SWAPIN; + HEAP_SPACE_VERIFIED(0); + if (is_non_value(result)) { + $FAIL($Fail); + } + $Dst = result; +} + +i_bs_get_utf8(Ctx, Fail, Dst) { + ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); + Eterm result = erts_bs_get_utf8(mb); + + if (is_non_value(result)) { + $FAIL($Fail); + } + $Dst = result; +} + +i_bs_get_utf16(Ctx, Fail, Flags, Dst) { + ErlBinMatchBuffer* mb = ms_matchbuffer($Ctx); + Eterm result = erts_bs_get_utf16(mb, $Flags); + + if (is_non_value(result)) { + $FAIL($Fail); + } + $Dst = result; +} + +bs_context_to_binary := ctx_to_bin.fetch.execute; +i_bs_get_binary_all_reuse := ctx_to_bin.fetch_bin.execute; + +ctx_to_bin.head() { + Eterm context; + ErlBinMatchBuffer* mb; + Uint size; + Uint offs; +} + +ctx_to_bin.fetch(Src) { + context = $Src; + if (is_boxed(context) && + header_is_bin_matchstate(*boxed_val(context))) { + ErlBinMatchState* ms; + ms = (ErlBinMatchState *) boxed_val(context); + mb = &ms->mb; + offs = ms->save_offset[0]; + size = mb->size - offs; + } else { + $NEXT0(); + } +} + +ctx_to_bin.fetch_bin(Src, Fail, Unit) { + context = $Src; + mb = ms_matchbuffer(context); + size = mb->size - mb->offset; + if (size % $Unit != 0) { + $FAIL($Fail); + } + offs = mb->offset; +} + +ctx_to_bin.execute() { + Uint hole_size; + Uint orig = mb->orig; + ErlSubBin* sb = (ErlSubBin *) boxed_val(context); + hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; + sb->thing_word = HEADER_SUB_BIN; + sb->size = BYTE_OFFSET(size); + sb->bitsize = BIT_OFFSET(size); + sb->offs = BYTE_OFFSET(offs); + sb->bitoffs = BIT_OFFSET(offs); + sb->is_writable = 0; + sb->orig = orig; + if (hole_size) { + sb[1].thing_word = make_pos_bignum_header(hole_size-1); + } +} + +i_bs_match_string(Ctx, Fail, Bits, Ptr) { + byte* bytes = (byte *) $Ptr; + Uint bits = $Bits; + ErlBinMatchBuffer* mb; + Uint offs; + + mb = ms_matchbuffer($Ctx); + if (mb->size - mb->offset < bits) { + $FAIL($Fail); + } + offs = mb->offset & 7; + if (offs == 0 && (bits & 7) == 0) { + if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) { + $FAIL($Fail); + } + } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) { + $FAIL($Fail); + } + mb->offset += bits; +} + +i_bs_save2(Src, Slot) { + ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + _ms->save_offset[$Slot] = _ms->mb.offset; +} + +i_bs_restore2(Src, Slot) { + ErlBinMatchState* _ms = (ErlBinMatchState*) boxed_val((Eterm) $Src); + _ms->mb.offset = _ms->save_offset[$Slot]; +} diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index ec6267711b..34e46f5f33 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -34,8 +34,8 @@ # define CIX_TRACE(text) #endif -erts_smp_atomic32_t the_active_code_index; -erts_smp_atomic32_t the_staging_code_index; +erts_atomic32_t the_active_code_index; +erts_atomic32_t the_staging_code_index; static Process* code_writing_process = NULL; struct code_write_queue_item { @@ -43,7 +43,7 @@ struct code_write_queue_item { struct code_write_queue_item* next; }; static struct code_write_queue_item* code_write_queue = NULL; -static erts_smp_mtx_t code_write_permission_mtx; +static erts_mtx_t code_write_permission_mtx; #ifdef ERTS_ENABLE_LOCK_CHECK static erts_tsd_key_t has_code_write_permission; @@ -55,9 +55,10 @@ void erts_code_ix_init(void) * single threaded with active and staging set both to zero. * Preloading is finished by a commit that will set things straight. */ - erts_smp_atomic32_init_nob(&the_active_code_index, 0); - erts_smp_atomic32_init_nob(&the_staging_code_index, 0); - erts_smp_mtx_init(&code_write_permission_mtx, "code_write_permission"); + erts_atomic32_init_nob(&the_active_code_index, 0); + erts_atomic32_init_nob(&the_staging_code_index, 0); + erts_mtx_init(&code_write_permission_mtx, "code_write_permission", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); #ifdef ERTS_ENABLE_LOCK_CHECK erts_tsd_key_create(&has_code_write_permission, "erts_has_code_write_permission"); @@ -90,9 +91,9 @@ void erts_commit_staging_code_ix(void) /* We need to this lock as we are now making the staging export table active */ export_staging_lock(); ix = erts_staging_code_ix(); - erts_smp_atomic32_set_nob(&the_active_code_index, ix); + erts_atomic32_set_nob(&the_active_code_index, ix); ix = (ix + 1) % ERTS_NUM_CODE_IX; - erts_smp_atomic32_set_nob(&the_staging_code_index, ix); + erts_atomic32_set_nob(&the_staging_code_index, ix); export_staging_unlock(); erts_tracer_nif_clear(); CIX_TRACE("activate"); @@ -114,12 +115,10 @@ void erts_abort_staging_code_ix(void) int erts_try_seize_code_write_permission(Process* c_p) { int success; -#ifdef ERTS_SMP - ASSERT(!erts_smp_thr_progress_is_blocking()); /* to avoid deadlock */ -#endif + ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */ ASSERT(c_p != NULL); - erts_smp_mtx_lock(&code_write_permission_mtx); + erts_mtx_lock(&code_write_permission_mtx); success = (code_writing_process == NULL); if (success) { code_writing_process = c_p; @@ -137,21 +136,21 @@ int erts_try_seize_code_write_permission(Process* c_p) code_write_queue = qitem; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); } - erts_smp_mtx_unlock(&code_write_permission_mtx); + erts_mtx_unlock(&code_write_permission_mtx); return success; } void erts_release_code_write_permission(void) { - erts_smp_mtx_lock(&code_write_permission_mtx); - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + erts_mtx_lock(&code_write_permission_mtx); + ERTS_LC_ASSERT(erts_has_code_write_permission()); while (code_write_queue != NULL) { /* unleash the entire herd */ struct code_write_queue_item* qitem = code_write_queue; - erts_smp_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(qitem->p)) { erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS); } - erts_smp_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS); code_write_queue = qitem->next; erts_proc_dec_refc(qitem->p); erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem); @@ -160,7 +159,7 @@ void erts_release_code_write_permission(void) #ifdef ERTS_ENABLE_LOCK_CHECK erts_tsd_set(has_code_write_permission, (void *) 0); #endif - erts_smp_mtx_unlock(&code_write_permission_mtx); + erts_mtx_unlock(&code_write_permission_mtx); } #ifdef ERTS_ENABLE_LOCK_CHECK diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index a28b0cd36e..42976d2301 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -176,7 +176,7 @@ int erts_has_code_write_permission(void); ERTS_GLB_INLINE BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) { - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op); ASSERT_MFA(&ci->mfa); return (BeamInstr*)(ci + 1); } @@ -185,7 +185,7 @@ ERTS_GLB_INLINE ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) { ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI) || !ci->op); ASSERT_MFA(&ci->mfa); return ci; } @@ -205,16 +205,16 @@ ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I) return mfa; } -extern erts_smp_atomic32_t the_active_code_index; -extern erts_smp_atomic32_t the_staging_code_index; +extern erts_atomic32_t the_active_code_index; +extern erts_atomic32_t the_staging_code_index; ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void) { - return erts_smp_atomic32_read_nob(&the_active_code_index); + return erts_atomic32_read_nob(&the_active_code_index); } ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void) { - return erts_smp_atomic32_read_nob(&the_staging_code_index); + return erts_atomic32_read_nob(&the_staging_code_index); } #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index fefde256d7..10bf197405 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -845,7 +845,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint funp = (ErlFunThing *) tp; funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; - erts_smp_refc_inc(&funp->fe->refc, 2); + erts_refc_inc(&funp->fe->refc, 2); *argp = make_fun(tp); } break; @@ -854,7 +854,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) objp; - erts_smp_refc_inc(&etp->node->refc, 2); + erts_refc_inc(&etp->node->refc, 2); } L_off_heap_node_container_common: { @@ -1531,7 +1531,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, } funp->next = off_heap->first; off_heap->first = (struct erl_off_heap_header*) funp; - erts_smp_refc_inc(&funp->fe->refc, 2); + erts_refc_inc(&funp->fe->refc, 2); goto cleanup_next; } case MAP_SUBTAG: @@ -1658,7 +1658,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) ptr; - erts_smp_refc_inc(&etp->node->refc, 2); + erts_refc_inc(&etp->node->refc, 2); } off_heap_node_container_common: { @@ -1855,7 +1855,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case FUN_SUBTAG: { ErlFunThing* funp = (ErlFunThing *) (tp-1); - erts_smp_refc_inc(&funp->fe->refc, 2); + erts_refc_inc(&funp->fe->refc, 2); } goto off_heap_common; case EXTERNAL_PID_SUBTAG: @@ -1863,7 +1863,7 @@ Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (tp-1); - erts_smp_refc_inc(&etp->node->refc, 2); + erts_refc_inc(&etp->node->refc, 2); } off_heap_common: { diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 982f1066df..bc168fc58d 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -121,7 +121,7 @@ Export* dexit_trap = NULL; Export* dmonitor_p_trap = NULL; /* local variables */ - +static Export *dist_ctrl_put_data_trap; /* forward declarations */ @@ -130,8 +130,8 @@ static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy); static void send_nodes_mon_msgs(Process *, Eterm, Eterm, Eterm, Eterm); static void init_nodes_monitors(void); -static erts_smp_atomic_t no_caches; -static erts_smp_atomic_t no_nodes; +static erts_atomic_t no_caches; +static erts_atomic_t no_nodes; struct { Eterm reason; @@ -144,8 +144,8 @@ delete_cache(ErtsAtomCache *cache) { if (cache) { erts_free(ERTS_ALC_T_DCACHE, (void *) cache); - ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0); - erts_smp_atomic_dec_nob(&no_caches); + ASSERT(erts_atomic_read_nob(&no_caches) > 0); + erts_atomic_dec_nob(&no_caches); } } @@ -156,14 +156,12 @@ create_cache(DistEntry *dep) int i; ErtsAtomCache *cp; - ERTS_SMP_LC_ASSERT( - is_internal_port(dep->cid) - && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); + ERTS_LC_ASSERT(is_nil(dep->cid)); ASSERT(!dep->cache); dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE, sizeof(ErtsAtomCache)); - erts_smp_atomic_inc_nob(&no_caches); + erts_atomic_inc_nob(&no_caches); for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) { cp->in_arr[i] = THE_NON_VALUE; cp->out_arr[i] = THE_NON_VALUE; @@ -172,15 +170,17 @@ create_cache(DistEntry *dep) Uint erts_dist_cache_size(void) { - return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache); + return (Uint) erts_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache); } static ErtsProcList * -get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) +get_suspended_on_de(DistEntry *dep, erts_aint32_t unset_qflgs) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock)); - dep->qflgs &= ~unset_qflgs; - if (dep->qflgs & ERTS_DE_QFLG_EXIT) { + erts_aint32_t qflgs; + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&dep->qlock)); + qflgs = erts_atomic32_read_band_acqb(&dep->qflgs, ~unset_qflgs); + qflgs &= ~unset_qflgs; + if (qflgs & ERTS_DE_QFLG_EXIT) { /* No resume when exit has been scheduled */ return NULL; } @@ -283,17 +283,15 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp) watched = (is_atom(rmon->name) ? TUPLE2(lhp, rmon->name, dep->sysname) : rmon->u.pid); -#ifdef ERTS_SMP rp_locks |= ERTS_PROC_LOCKS_MSG_SEND; - erts_smp_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND); -#endif + erts_proc_lock(rp, ERTS_PROC_LOCKS_MSG_SEND); erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, watched, am_noconnection); erts_destroy_monitor(rmon); } UnUseTmpHeapNoproc(3); } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); done: erts_destroy_monitor(mon); } @@ -342,7 +340,7 @@ static void doit_link_net_exits_sub(ErtsLink *sublnk, void *vlnecp) trace_proc(NULL, 0, rp, am_getting_unlinked, sublnk->pid); } } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } done: erts_destroy_link(sublnk); @@ -384,7 +382,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) rp = erts_proc_lookup(lnk->pid); if (!rp) goto done; - erts_smp_proc_lock(rp, rp_locks); + erts_proc_lock(rp, rp_locks); rlnk = erts_remove_link(&ERTS_P_LINKS(rp), name); if (rlnk != NULL) { ASSERT(is_atom(rlnk->pid) && (rlnk->type == LINK_NODE)); @@ -401,7 +399,7 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp) tup = TUPLE2(hp, am_nodedown, name); erts_queue_message(rp, rp_locks, msgp, tup, am_system); } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } done: erts_destroy_link(lnk); @@ -413,16 +411,16 @@ set_node_not_alive(void *unused) ErlHeapFragment *bp; Eterm nodename = erts_this_dist_entry->sysname; - ASSERT(erts_smp_atomic_read_nob(&no_nodes) == 0); + ASSERT(erts_atomic_read_nob(&no_nodes) == 0); - erts_smp_thr_progress_block(); + erts_thr_progress_block(); erts_set_this_node(am_Noname, 0); erts_is_alive = 0; send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nodedown.reason); nodedown.reason = NIL; bp = nodedown.bp; nodedown.bp = NULL; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); if (bp) free_message_buffer(bp); } @@ -430,7 +428,7 @@ set_node_not_alive(void *unused) static ERTS_INLINE void dec_no_nodes(void) { - erts_aint_t no = erts_smp_atomic_dec_read_mb(&no_nodes); + erts_aint_t no = erts_atomic_dec_read_mb(&no_nodes); ASSERT(no >= 0); ASSERT(erts_get_scheduler_id()); /* Need to be a scheduler */ if (no == 0) @@ -443,12 +441,40 @@ static ERTS_INLINE void inc_no_nodes(void) { #ifdef DEBUG - erts_aint_t no = erts_smp_atomic_read_nob(&no_nodes); + erts_aint_t no = erts_atomic_read_nob(&no_nodes); ASSERT(erts_is_alive ? no > 0 : no == 0); #endif - erts_smp_atomic_inc_mb(&no_nodes); + erts_atomic_inc_mb(&no_nodes); } - + +static void +kill_dist_ctrl_proc(void *vpid) +{ + Eterm pid = (Eterm) vpid; + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + Process *rp = erts_pid2proc(NULL, 0, pid, rp_locks); + if (rp) { + erts_send_exit_signal(NULL, rp->common.id, rp, &rp_locks, + am_kill, NIL, NULL, 0); + if (rp_locks) + erts_proc_unlock(rp, rp_locks); + } +} + +static void +schedule_kill_dist_ctrl_proc(Eterm pid) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int sched_id = 1; + if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp)) + sched_id = 1; + else + sched_id = (int) esdp->no; + erts_schedule_misc_aux_work(sched_id, + kill_dist_ctrl_proc, + (void *) (UWord) pid); +} + /* * proc is currently running or exiting process. */ @@ -458,58 +484,62 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep == erts_this_dist_entry) { /* Net kernel has died (clean up!!) */ DistEntry *tdep; - int no_dist_port = 0; + int no_dist_ctrl = 0; Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) - no_dist_port++; + no_dist_ctrl++; for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) - no_dist_port++; + no_dist_ctrl++; /* KILL all port controllers */ - if (no_dist_port == 0) - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + if (no_dist_ctrl == 0) + erts_rwmtx_runlock(&erts_dist_table_rwmtx); else { Eterm def_buf[128]; int i = 0; - Eterm *dist_port; + Eterm *dist_ctrl; - if (no_dist_port <= sizeof(def_buf)/sizeof(def_buf[0])) - dist_port = &def_buf[0]; + if (no_dist_ctrl <= sizeof(def_buf)/sizeof(def_buf[0])) + dist_ctrl = &def_buf[0]; else - dist_port = erts_alloc(ERTS_ALC_T_TMP, - sizeof(Eterm)*no_dist_port); + dist_ctrl = erts_alloc(ERTS_ALC_T_TMP, + sizeof(Eterm)*no_dist_ctrl); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) { - ASSERT(is_internal_port(tdep->cid)); - dist_port[i++] = tdep->cid; + ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + dist_ctrl[i++] = tdep->cid; } for (tdep = erts_visible_dist_entries; tdep; tdep = tdep->next) { - ASSERT(is_internal_port(tdep->cid)); - dist_port[i++] = tdep->cid; + ASSERT(is_internal_port(tdep->cid) || is_internal_pid(tdep->cid)); + dist_ctrl[i++] = tdep->cid; } - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); - - for (i = 0; i < no_dist_port; i++) { - Port *prt = erts_port_lookup(dist_port[i], - ERTS_PORT_SFLGS_INVALID_LOOKUP); - if (!prt) - continue; - ASSERT(erts_atomic32_read_nob(&prt->state) - & ERTS_PORT_SFLG_DISTRIBUTION); - - erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED, - prt, dist_port[i], nd_reason, NULL); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); + + for (i = 0; i < no_dist_ctrl; i++) { + if (is_internal_pid(dist_ctrl[i])) + schedule_kill_dist_ctrl_proc(dist_ctrl[i]); + else { + Port *prt = erts_port_lookup(dist_ctrl[i], + ERTS_PORT_SFLGS_INVALID_LOOKUP); + if (prt) { + ASSERT(erts_atomic32_read_nob(&prt->state) + & ERTS_PORT_SFLG_DISTRIBUTION); + + erts_port_exit(NULL, ERTS_PORT_SIG_FLG_FORCE_SCHED, + prt, dist_ctrl[i], nd_reason, NULL); + } + } } - if (dist_port != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, dist_port); + if (dist_ctrl != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, dist_ctrl); } /* - * When last dist port exits, node will be taken + * When last dist ctrl exits, node will be taken * from alive to not alive. */ ASSERT(is_nil(nodedown.reason) && !nodedown.bp); @@ -526,52 +556,51 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) &nodedown.bp->off_heap); } } - else { /* Call from distribution port */ + else { /* Call from distribution controller (port/process) */ NetExitsContext nec = {dep}; ErtsLink *nlinks; ErtsLink *node_links; ErtsMonitor *monitors; Uint32 flags; - erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1); - erts_smp_de_rwlock(dep); + erts_atomic_set_mb(&dep->dist_cmd_scheduled, 1); + erts_de_rwlock(dep); - ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid) - && erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); + if (is_internal_port(dep->cid)) { + ERTS_LC_ASSERT(erts_lc_is_port_locked(erts_port_lookup_raw(dep->cid))); - if (erts_port_task_is_scheduled(&dep->dist_cmd)) - erts_port_task_abort(&dep->dist_cmd); + if (erts_port_task_is_scheduled(&dep->dist_cmd)) + erts_port_task_abort(&dep->dist_cmd); + } if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT); - erts_smp_mtx_unlock(&dep->qlock); + ASSERT(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT); #endif } else { dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_mtx_lock(&dep->qlock); - ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); - dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_lock(&dep->qlock); + ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); + erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_EXIT); + erts_mtx_unlock(&dep->qlock); } - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); monitors = dep->monitors; nlinks = dep->nlinks; node_links = dep->node_links; dep->monitors = NULL; dep->nlinks = NULL; dep->node_links = NULL; - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); nodename = dep->sysname; flags = dep->flags; erts_set_dist_entry_not_connected(dep); - erts_smp_de_rwunlock(dep); + erts_de_rwunlock(dep); erts_sweep_monitors(monitors, &doit_monitor_net_exits, (void *) &nec); erts_sweep_links(nlinks, &doit_link_net_exits, (void *) &nec); @@ -605,8 +634,8 @@ void init_dist(void) nodedown.reason = NIL; nodedown.bp = NULL; - erts_smp_atomic_init_nob(&no_nodes, 0); - erts_smp_atomic_init_nob(&no_caches, 0); + erts_atomic_init_nob(&no_nodes, 0); + erts_atomic_init_nob(&no_caches, 0); /* Lookup/Install all references to trap functions */ dsend2_trap = trap_function(am_dsend,2); @@ -618,6 +647,9 @@ void init_dist(void) dgroup_leader_trap = trap_function(am_dgroup_leader,2); dexit_trap = trap_function(am_dexit, 2); dmonitor_p_trap = trap_function(am_dmonitor_p, 2); + dist_ctrl_put_data_trap = erts_export_put(am_erts_internal, + am_dist_ctrl_put_data, + 2); } #define ErtsDistOutputBuf2Binary(OB) \ @@ -659,19 +691,24 @@ static void clear_dist_entry(DistEntry *dep) ErtsProcList *suspendees; ErtsDistOutputBuf *obuf; - erts_smp_de_rwlock(dep); + erts_de_rwlock(dep); + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) NIL); cache = dep->cache; dep->cache = NULL; #ifdef DEBUG - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); ASSERT(!dep->nlinks); ASSERT(!dep->node_links); ASSERT(!dep->monitors); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); #endif - erts_smp_mtx_lock(&dep->qlock); + erts_mtx_lock(&dep->qlock); + + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; @@ -680,17 +717,24 @@ static void clear_dist_entry(DistEntry *dep) obuf = dep->out_queue.first; } + if (dep->tmp_out_queue.first) { + dep->tmp_out_queue.last->next = obuf; + obuf = dep->tmp_out_queue.first; + } + dep->out_queue.first = NULL; dep->out_queue.last = NULL; + dep->tmp_out_queue.first = NULL; + dep->tmp_out_queue.last = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; dep->status = 0; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - erts_smp_mtx_unlock(&dep->qlock); - erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0); + erts_mtx_unlock(&dep->qlock); + erts_atomic_set_nob(&dep->dist_cmd_scheduled, 0); dep->send = NULL; - erts_smp_de_rwunlock(dep); + erts_de_rwunlock(dep); erts_resume_processes(suspendees); @@ -705,10 +749,11 @@ static void clear_dist_entry(DistEntry *dep) } if (obufsize) { - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_lock(&dep->qlock); + ASSERT(erts_atomic_read_nob(&dep->qsize) >= obufsize); + erts_atomic_add_nob(&dep->qsize, + (erts_aint_t) -obufsize); + erts_mtx_unlock(&dep->qlock); } } @@ -813,9 +858,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched, watched, watcher, ref, reason); #ifdef DEBUG - erts_smp_de_links_lock(dsdp->dep); + erts_de_links_lock(dsdp->dep); ASSERT(!erts_lookup_monitor(dsdp->dep->monitors, ref)); - erts_smp_de_links_unlock(dsdp->dep); + erts_de_links_unlock(dsdp->dep); #endif res = dsig_send_ctl(dsdp, ctl, 1); @@ -906,11 +951,30 @@ erts_dsig_send_msg(Eterm remote, Eterm message, ErtsSendContext* ctx) } #endif - if (token != NIL) - ctl = TUPLE4(&ctx->ctl_heap[0], - make_small(DOP_SEND_TT), am_Empty, remote, token); - else - ctl = TUPLE3(&ctx->ctl_heap[0], make_small(DOP_SEND), am_Empty, remote); + if (token != NIL) { + Eterm el1, el2; + if (ctx->dep->flags & DFLAG_SEND_SENDER) { + el1 = make_small(DOP_SEND_SENDER_TT); + el2 = sender->common.id; + } + else { + el1 = make_small(DOP_SEND_TT); + el2 = am_Empty; + } + ctl = TUPLE4(&ctx->ctl_heap[0], el1, el2, remote, token); + } + else { + Eterm el1, el2; + if (ctx->dep->flags & DFLAG_SEND_SENDER) { + el1 = make_small(DOP_SEND_SENDER); + el2 = sender->common.id; + } + else { + el1 = make_small(DOP_SEND); + el2 = am_Empty; + } + ctl = TUPLE3(&ctx->ctl_heap[0], el1, el2, remote); + } DTRACE6(message_send, sender_name, receiver_name, msize, tok_label, tok_lastcnt, tok_serial); DTRACE7(message_send_remote, sender_name, node_name, receiver_name, @@ -1147,22 +1211,25 @@ int erts_net_message(Port *prt, ErtsLink *lnk; Uint tuple_arity; int res; + Uint32 connection_id; #ifdef ERTS_DIST_MSG_DBG ErlDrvSizeT orig_len = len; #endif UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); if (!erts_is_alive) { UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; } - if (hlen != 0) - goto data_error; + + + ASSERT(hlen == 0); + if (len == 0) { /* HANDLE TICK !!! */ UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); return 0; @@ -1181,30 +1248,31 @@ int erts_net_message(Port *prt, len--; } - if (len == 0) { - PURIFY_MSG("data error"); - goto data_error; - } + res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache, &connection_id); - res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache); - - if (res >= 0) - res = ctl_len = erts_decode_dist_ext_size(&ede); - else { + switch (res) { + case ERTS_PREP_DIST_EXT_CLOSED: + return 0; /* Connection not alive; ignore signal... */ + case ERTS_PREP_DIST_EXT_FAILED: #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n"); bw(buf, orig_len); #endif - ctl_len = 0; - } - - if (res < 0) { + goto data_error; + case ERTS_PREP_DIST_EXT_SUCCESS: + ctl_len = erts_decode_dist_ext_size(&ede); + if (ctl_len < 0) { #ifdef ERTS_DIST_MSG_DBG - erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); - bw(buf, orig_len); + erts_fprintf(stderr, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); + bw(buf, orig_len); #endif - PURIFY_MSG("data error"); - goto data_error; + PURIFY_MSG("data error"); + goto data_error; + } + break; + default: + ERTS_INTERNAL_ERROR("Unexpected result from erts_prepare_dist_ext()"); + break; } if (ctl_len > DIST_CTL_DEFAULT_SIZE) { @@ -1235,6 +1303,7 @@ int erts_net_message(Port *prt, } token_size = 0; + token = NIL; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: @@ -1263,23 +1332,23 @@ int erts_net_message(Port *prt, break; } - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); res = erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, from); if (res < 0) { /* It was already there! Lets skip the rest... */ - erts_smp_de_links_unlock(dep); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); break; } lnk = erts_add_or_lookup_link(&(dep->nlinks), LINK_PID, rp->common.id); erts_add_link(&(ERTS_LINK_ROOT(lnk)), LINK_PID, from); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (IS_TRACED_FL(rp, F_TRACE_PROCS)) trace_proc(NULL, 0, rp, am_getting_linked, from); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); break; case DOP_UNLINK: { @@ -1305,7 +1374,7 @@ int erts_net_message(Port *prt, trace_proc(NULL, 0, rp, am_getting_unlinked, from); } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); erts_remove_dist_link(&dld, to, from, dep); erts_destroy_dist_link(&dld); @@ -1357,11 +1426,11 @@ int erts_net_message(Port *prt, else { if (is_atom(watched)) watched = rp->common.id; - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); erts_add_monitor(&(dep->monitors), MON_ORIGIN, ref, watched, name); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, watcher, name); - erts_smp_de_links_unlock(dep); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } break; @@ -1383,9 +1452,9 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); mon = erts_remove_monitor(&(dep->monitors),ref); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); /* ASSERT(mon != NULL); can happen in case of broken dist message */ if (mon == NULL) { break; @@ -1399,7 +1468,7 @@ int erts_net_message(Port *prt, break; } mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); ASSERT(mon != NULL); if (mon == NULL) { break; @@ -1460,42 +1529,56 @@ int erts_net_message(Port *prt, erts_queue_dist_message(rp, locks, ede_copy, token, from); if (locks) - erts_smp_proc_unlock(rp, locks); + erts_proc_unlock(rp, locks); } break; + case DOP_SEND_SENDER_TT: { + Uint xsize; case DOP_SEND_TT: + if (tuple_arity != 4) { goto invalid_message; } - - token_size = size_object(tuple[4]); - /* Fall through ... */ + + token = tuple[4]; + token_size = size_object(token); + xsize = ERTS_HEAP_FRAG_SIZE(token_size); + goto send_common; + + case DOP_SEND_SENDER: case DOP_SEND: + + token = NIL; + xsize = 0; + if (tuple_arity != 3) + goto invalid_message; + + send_common: + /* - * There is intentionally no testing of the cookie (it is always '') - * from R9B and onwards. + * If DOP_SEND_SENDER or DOP_SEND_SENDER_TT element 2 contains + * the sender pid (i.e. DFLAG_SEND_SENDER is set); otherwise, + * the atom '' (empty cookie). */ + ASSERT((type == DOP_SEND_SENDER || type == DOP_SEND_SENDER_TT) + ? (is_pid(tuple[2]) && (dep->flags & DFLAG_SEND_SENDER)) + : tuple[2] == am_Empty); + #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif - if (type != DOP_SEND_TT && tuple_arity != 3) { - goto invalid_message; - } to = tuple[3]; if (is_not_pid(to)) { goto invalid_message; } rp = erts_proc_lookup(to); if (rp) { - Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); ErtsProcLocks locks = 0; ErtsDistExternal *ede_copy; ede_copy = erts_make_dist_ext_copy(&ede, xsize); - if (type == DOP_SEND) { - token = NIL; - } else { + if (is_not_nil(token)) { ErlHeapFragment *heap_frag; ErlOffHeap *ohp; ASSERT(xsize); @@ -1503,15 +1586,15 @@ int erts_net_message(Port *prt, ERTS_INIT_HEAP_FRAG(heap_frag, token_size, token_size); hp = heap_frag->mem; ohp = &heap_frag->off_heap; - token = tuple[4]; token = copy_struct(token, token_size, &hp, ohp); } - erts_queue_dist_message(rp, locks, ede_copy, token, tuple[2]); + erts_queue_dist_message(rp, locks, ede_copy, token, am_Empty); if (locks) - erts_smp_proc_unlock(rp, locks); + erts_proc_unlock(rp, locks); } break; + } case DOP_MONITOR_P_EXIT: { /* We are monitoring a process on the remote node which dies, we get @@ -1535,7 +1618,7 @@ int erts_net_message(Port *prt, goto invalid_message; } - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); sysname = dep->sysname; mon = erts_remove_monitor(&(dep->monitors), ref); /* @@ -1544,7 +1627,7 @@ int erts_net_message(Port *prt, * removed info about monitor. In this case, do nothing * and everything will be as it should. */ - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (mon == NULL) { break; } @@ -1558,7 +1641,7 @@ int erts_net_message(Port *prt, mon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); if (mon == NULL) { - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); break; } UseTmpHeapNoproc(3); @@ -1569,7 +1652,7 @@ int erts_net_message(Port *prt, erts_queue_monitor_message(rp, &rp_locks, ref, am_process, watched, reason); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_destroy_monitor(mon); UnUseTmpHeapNoproc(3); break; @@ -1631,13 +1714,13 @@ int erts_net_message(Port *prt, if (xres >= 0 && IS_TRACED_FL(rp, F_TRACE_PROCS)) { /* We didn't exit the process and it is traced */ if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; } trace_proc(NULL, 0, rp, am_getting_unlinked, from); } } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } erts_remove_dist_link(&dld, to, from, dep); if (lnk) @@ -1679,7 +1762,7 @@ int erts_net_message(Port *prt, token, NULL, 0); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } break; } @@ -1697,7 +1780,7 @@ int erts_net_message(Port *prt, if (!rp) break; rp->group_leader = STORE_NC_IN_PROC(rp, from); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); break; default: @@ -1709,7 +1792,7 @@ int erts_net_message(Port *prt, erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; return 0; invalid_message: { @@ -1725,8 +1808,8 @@ decode_error: } data_error: UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); - erts_deliver_port_exit(prt, dep->cid, am_killed, 0, 1); - ERTS_SMP_CHK_NO_PROC_LOCKS; + erts_kill_dist_connection(dep, connection_id); + ERTS_CHK_NO_PROC_LOCKS; return -1; } @@ -1746,6 +1829,31 @@ static int dsig_send_ctl(ErtsDSigData* dsdp, Eterm ctl, int force_busy) return ret; } +static ERTS_INLINE void +notify_dist_data(Process *c_p, Eterm pid) +{ + Process *rp; + ErtsProcLocks rp_locks; + + ASSERT(erts_get_scheduler_data() + && !ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ASSERT(is_internal_pid(pid)); + + if (c_p && c_p->common.id == pid) { + rp = c_p; + rp_locks = ERTS_PROC_LOCK_MAIN; + } + else { + rp = erts_proc_lookup(pid); + rp_locks = 0; + } + + if (rp) { + ErtsMessage *mp = erts_alloc_message(0, NULL); + erts_queue_message(rp, rp_locks, mp, am_dist_data, am_system); + } +} + int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) { @@ -1762,7 +1870,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) if (!ctx->c_p || dsdp->no_suspend) ctx->force_busy = 1; - ERTS_SMP_LC_ASSERT(!ctx->c_p + ERTS_LC_ASSERT(!ctx->c_p || (ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(ctx->c_p))); @@ -1851,28 +1959,48 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) * and if so enqueue the signal and schedule it for send. */ ctx->obuf->next = NULL; - erts_smp_de_rlock(dep); + erts_de_rlock(dep); cid = dep->cid; if (cid != dsdp->cid || dep->connection_id != dsdp->connection_id || dep->status & ERTS_DE_SFLG_EXITING) { /* Not the same connection as when we started; drop message... */ - erts_smp_de_runlock(dep); + erts_de_runlock(dep); free_dist_obuf(ctx->obuf); } else { + Sint qsize; + erts_aint32_t qflgs; ErtsProcList *plp = NULL; - erts_smp_mtx_lock(&dep->qlock); - dep->qsize += size_obuf(ctx->obuf); - if (dep->qsize >= erts_dist_buf_busy_limit) - dep->qflgs |= ERTS_DE_QFLG_BUSY; - if (!ctx->force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_mtx_unlock(&dep->qlock); + Eterm notify_proc = NIL; + Sint obsz = size_obuf(ctx->obuf); + + erts_mtx_lock(&dep->qlock); + qsize = erts_atomic_add_read_nob(&dep->qsize, (erts_aint_t) obsz); + ASSERT(qsize >= obsz); + qflgs = erts_atomic32_read_nob(&dep->qflgs); + if (!(qflgs & ERTS_DE_QFLG_BUSY) && qsize >= erts_dist_buf_busy_limit) { + erts_atomic32_read_bor_relb(&dep->qflgs, ERTS_DE_QFLG_BUSY); + qflgs |= ERTS_DE_QFLG_BUSY; + } + if (qsize == obsz && (qflgs & ERTS_DE_QFLG_REQ_INFO)) { + /* Previously empty queue and info requested... */ + qflgs = erts_atomic32_read_band_mb(&dep->qflgs, + ~ERTS_DE_QFLG_REQ_INFO); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) { + notify_proc = dep->cid; + ASSERT(is_internal_pid(notify_proc)); + } + /* else: requester will send itself the message... */ + qflgs &= ~ERTS_DE_QFLG_REQ_INFO; + } + if (!ctx->force_busy && (qflgs & ERTS_DE_QFLG_BUSY)) { + erts_mtx_unlock(&dep->qlock); plp = erts_proclist_create(ctx->c_p); erts_suspend(ctx->c_p, ERTS_PROC_LOCK_MAIN, NULL); suspended = 1; - erts_smp_mtx_lock(&dep->qlock); + erts_mtx_lock(&dep->qlock); } /* Enqueue obuf on dist entry */ @@ -1883,7 +2011,8 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) dep->out_queue.last = ctx->obuf; if (!ctx->force_busy) { - if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) { + qflgs = erts_atomic32_read_nob(&dep->qflgs); + if (!(qflgs & ERTS_DE_QFLG_BUSY)) { if (suspended) resume = 1; /* was busy when we started, but isn't now */ #ifdef USE_VM_PROBES @@ -1907,9 +2036,12 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) } } - erts_smp_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); - erts_smp_de_runlock(dep); + erts_mtx_unlock(&dep->qlock); + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); + erts_de_runlock(dep); + if (is_internal_pid(notify_proc)) + notify_dist_data(ctx->c_p, notify_proc); if (resume) { erts_resume(ctx->c_p, ERTS_PROC_LOCK_MAIN); @@ -1963,16 +2095,20 @@ static Uint dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - Uint size = obuf->ext_endp - obuf->extp; + ErlDrvSizeT size; + char *bufp; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - if (size > (Uint) INT_MAX) - erts_exit(ERTS_DUMP_EXIT, - "Absurdly large distribution output data buffer " - "(%beu bytes) passed.\n", - size); + if (!obuf) { + size = 0; + bufp = NULL; + } + else { + size = obuf->ext_endp - obuf->extp; + bufp = (char*) obuf->extp; + } #ifdef USE_VM_PROBES if (DTRACE_ENABLED(dist_output)) { @@ -1987,11 +2123,10 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) remote_str, size); } #endif + prt->caller = NIL; fpe_was_unmasked = erts_block_fpe(); - (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, - (char*) obuf->extp, - (int) size); + (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data, bufp, size); erts_unblock_fpe(fpe_was_unmasked); return size; } @@ -2000,33 +2135,41 @@ static Uint dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) { int fpe_was_unmasked; - Uint size = obuf->ext_endp - obuf->extp; + ErlDrvSizeT size; SysIOVec iov[2]; ErlDrvBinary* bv[2]; ErlIOVec eiov; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - if (size > (Uint) INT_MAX) - erts_exit(ERTS_DUMP_EXIT, - "Absurdly large distribution output data buffer " - "(%beu bytes) passed.\n", - size); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); iov[0].iov_base = NULL; iov[0].iov_len = 0; bv[0] = NULL; - iov[1].iov_base = obuf->extp; - iov[1].iov_len = size; - bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + if (!obuf) { + size = 0; + eiov.vsize = 1; + } + else { + size = obuf->ext_endp - obuf->extp; + eiov.vsize = 2; + + iov[1].iov_base = obuf->extp; + iov[1].iov_len = size; + bv[1] = Binary2ErlDrvBinary(ErtsDistOutputBuf2Binary(obuf)); + } - eiov.vsize = 2; eiov.size = size; eiov.iov = iov; eiov.binv = bv; + if (size > (Uint) INT_MAX) + erts_exit(ERTS_DUMP_EXIT, + "Absurdly large distribution output data buffer " + "(%beu bytes) passed.\n", + size); + ASSERT(prt->drv_ptr->outputv); #ifdef USE_VM_PROBES @@ -2074,29 +2217,25 @@ erts_dist_command(Port *prt, int reds_limit) Sint reds = ERTS_PORT_REDS_DIST_CMD_START; Uint32 status; Uint32 flags; - Sint obufsize = 0; + Sint qsize, obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); erts_aint32_t sched_flags; ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - - erts_smp_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be - removed if port command fails */ + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); - erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0); + erts_atomic_set_mb(&dep->dist_cmd_scheduled, 0); - erts_smp_de_rlock(dep); + erts_de_rlock(dep); flags = dep->flags; status = dep->status; send = dep->send; - erts_smp_de_runlock(dep); + erts_de_runlock(dep); if (status & ERTS_DE_SFLG_EXITING) { erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); - erts_deref_dist_entry(dep); return reds + ERTS_PORT_REDS_DIST_CMD_EXIT; } @@ -2110,19 +2249,19 @@ erts_dist_command(Port *prt, int reds_limit) * a mess. */ - erts_smp_mtx_lock(&dep->qlock); + erts_mtx_lock(&dep->qlock); oq.first = dep->out_queue.first; oq.last = dep->out_queue.last; dep->out_queue.first = NULL; dep->out_queue.last = NULL; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_unlock(&dep->qlock); foq.first = dep->finalized_out_queue.first; foq.last = dep->finalized_out_queue.last; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (reds > reds_limit) goto preempted; @@ -2130,21 +2269,21 @@ erts_dist_command(Port *prt, int reds_limit) if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) && foq.first) { int preempt = 0; do { - Uint size; - ErtsDistOutputBuf *fob; - - size = (*send)(prt, foq.first); - esdp->io.out += (Uint64) size; + Uint size; + ErtsDistOutputBuf *fob; + size = (*send)(prt, foq.first); + erts_atomic64_inc_nob(&dep->out); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(foq.first->extp, size); + erts_fprintf(stderr, ">> "); + bw(foq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = foq.first; - obufsize += size_obuf(fob); - foq.first = foq.first->next; - free_dist_obuf(fob); - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = foq.first; + obufsize += size_obuf(fob); + foq.first = foq.first->next; + free_dist_obuf(fob); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); if (sched_flags & ERTS_PTS_FLG_BUSY_PORT) break; @@ -2204,32 +2343,34 @@ erts_dist_command(Port *prt, int reds_limit) } } else { + int de_busy; int preempt = 0; while (oq.first && !preempt) { - ErtsDistOutputBuf *fob; - Uint size; - oq.first->extp - = erts_encode_ext_dist_header_finalize(oq.first->extp, - dep->cache, - flags); - reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; - if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) - *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' - needed */ - ASSERT(&oq.first->data[0] <= oq.first->extp - && oq.first->extp < oq.first->ext_endp); - size = (*send)(prt, oq.first); - esdp->io.out += (Uint64) size; + ErtsDistOutputBuf *fob; + Uint size; + oq.first->extp + = erts_encode_ext_dist_header_finalize(oq.first->extp, + dep->cache, + flags); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--oq.first->extp = PASS_THROUGH; /* Old node; 'pass through' + needed */ + ASSERT(&oq.first->data[0] <= oq.first->extp + && oq.first->extp < oq.first->ext_endp); + size = (*send)(prt, oq.first); + erts_atomic64_inc_nob(&dep->out); + esdp->io.out += (Uint64) size; #ifdef ERTS_RAW_DIST_MSG_DBG - erts_fprintf(stderr, ">> "); - bw(oq.first->extp, size); + erts_fprintf(stderr, ">> "); + bw(oq.first->extp, size); #endif - reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); - fob = oq.first; - obufsize += size_obuf(fob); - oq.first = oq.first->next; - free_dist_obuf(fob); - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + reds += ERTS_PORT_REDS_DIST_CMD_DATA(size); + fob = oq.first; + obufsize += size_obuf(fob); + oq.first = oq.first->next; + free_dist_obuf(fob); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); preempt = reds > reds_limit || (sched_flags & ERTS_PTS_FLG_EXIT); if ((sched_flags & ERTS_PTS_FLG_BUSY_PORT) && oq.first && !preempt) goto finalize_only; @@ -2256,23 +2397,24 @@ erts_dist_command(Port *prt, int reds_limit) * dist entry in a non-busy state and resume suspended * processes. */ - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; + erts_mtx_lock(&dep->qlock); + de_busy = !!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_BUSY); + qsize = (Sint) erts_atomic_add_read_nob(&dep->qsize, + (erts_aint_t) -obufsize); + ASSERT(qsize >= 0); obufsize = 0; if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT) - && (dep->qflgs & ERTS_DE_QFLG_BUSY) - && dep->qsize < erts_dist_buf_busy_limit) { + && de_busy && qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; int resumed; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_unlock(&dep->qlock); resumed = erts_resume_processes(suspendees); reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; } else - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_unlock(&dep->qlock); } ASSERT(!oq.first && !oq.last); @@ -2281,10 +2423,15 @@ erts_dist_command(Port *prt, int reds_limit) if (obufsize != 0) { ASSERT(obufsize > 0); - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qsize >= obufsize); - dep->qsize -= obufsize; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_lock(&dep->qlock); +#ifdef DEBUG + qsize = (Sint) erts_atomic_add_read_nob(&dep->qsize, + (erts_aint_t) -obufsize); + ASSERT(qsize >= 0); +#else + erts_atomic_add_nob(&dep->qsize, (erts_aint_t) -obufsize); +#endif + erts_mtx_unlock(&dep->qlock); } ASSERT(foq.first || !foq.last); @@ -2301,8 +2448,6 @@ erts_dist_command(Port *prt, int reds_limit) if (reds > INT_MAX/2) reds = INT_MAX/2; - erts_deref_dist_entry(dep); - return reds; preempted: @@ -2338,9 +2483,9 @@ erts_dist_command(Port *prt, int reds_limit) foq.last = NULL; #ifdef DEBUG - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qsize == obufsize); - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_lock(&dep->qlock); + ASSERT(erts_atomic_read_nob(&dep->qsize) == obufsize); + erts_mtx_unlock(&dep->qlock); #endif } else { @@ -2349,14 +2494,14 @@ erts_dist_command(Port *prt, int reds_limit) * Unhandle buffers need to be put back first * in out_queue. */ - erts_smp_mtx_lock(&dep->qlock); - dep->qsize -= obufsize; + erts_mtx_lock(&dep->qlock); + erts_atomic_add_nob(&dep->qsize, -obufsize); obufsize = 0; oq.last->next = dep->out_queue.first; dep->out_queue.first = oq.first; if (!dep->out_queue.last) dep->out_queue.last = oq.last; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_unlock(&dep->qlock); } erts_schedule_dist_command(prt, NULL); @@ -2364,6 +2509,370 @@ erts_dist_command(Port *prt, int reds_limit) goto done; } +#if 0 + +int +dist_data_finalize(Process *c_p, int reds_limit) +{ + int reds = 5; + DistEntry *dep = ; + ErtsDistOutputQueue oq, foq; + ErtsDistOutputBuf *ob; + int preempt; + + + erts_mtx_lock(&dep->qlock); + flags = dep->flags; + oq.first = dep->out_queue.first; + oq.last = dep->out_queue.last; + dep->out_queue.first = NULL; + dep->out_queue.last = NULL; + erts_mtx_unlock(&dep->qlock); + + if (!oq.first) { + ASSERT(!oq.last); + oq.first = dep->tmp_out_queue.first; + oq.last = dep->tmp_out_queue.last; + } + else { + ErtsDistOutputBuf *f, *l; + ASSERT(oq.last); + if (dep->tmp_out_queue.last) { + dep->tmp_out_queue.last->next = oq.first; + oq.first = dep->tmp_out_queue.first; + } + } + + if (!oq.first) { + /* Nothing to do... */ + ASSERT(!oq.last); + return reds; + } + + foq.first = dep->finalized_out_queue.first; + foq.last = dep->finalized_out_queue.last; + + preempt = 0; + ob = oq.first; + ASSERT(ob); + + do { + ob->extp = erts_encode_ext_dist_header_finalize(ob->extp, + dep->cache, + flags); + if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--ob->extp = PASS_THROUGH; /* Old node; 'pass through' + needed */ + ASSERT(&ob->data[0] <= ob->extp && ob->extp < ob->ext_endp); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + preempt = reds > reds_limit; + if (preempt) + break; + ob = ob->next; + } while (ob); + /* + * At least one buffer was finalized; if we got preempted, + * ob points to the last buffer that we finalized. + */ + if (foq.last) + foq.last->next = oq.first; + else + foq.first = oq.first; + if (!preempt) { + /* All buffers finalized */ + foq.last = oq.last; + oq.first = oq.last = NULL; + } + else { + /* Not all buffers finalized; split oq. */ + foq.last = ob; + oq.first = ob->next; + if (oq.first) + ob->next = NULL; + else + oq.last = NULL; + } + + dep->finalized_out_queue.first = foq.first; + dep->finalized_out_queue.last = foq.last; + dep->tmp_out_queue.first = oq.first; + dep->tmp_out_queue.last = oq.last; + + return reds; +} + +#endif + +BIF_RETTYPE +dist_ctrl_get_data_notification_1(BIF_ALIST_1) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + erts_aint32_t qflgs; + erts_aint_t qsize; + Eterm receiver = NIL; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + /* + * Caller is the only one that can consume from this queue + * and the only one that can set the req-info flag... + */ + + erts_de_rlock(dep); + + ASSERT(dep->cid == BIF_P->common.id); + + qflgs = erts_atomic32_read_acqb(&dep->qflgs); + + if (!(qflgs & ERTS_DE_QFLG_REQ_INFO)) { + qsize = erts_atomic_read_acqb(&dep->qsize); + ASSERT(qsize >= 0); + if (qsize > 0) + receiver = BIF_P->common.id; /* Notify ourselves... */ + else { /* Empty queue; set req-info flag... */ + qflgs = erts_atomic32_read_bor_mb(&dep->qflgs, + ERTS_DE_QFLG_REQ_INFO); + qsize = erts_atomic_read_acqb(&dep->qsize); + ASSERT(qsize >= 0); + if (qsize > 0) { + qflgs = erts_atomic32_read_band_mb(&dep->qflgs, + ~ERTS_DE_QFLG_REQ_INFO); + if (qflgs & ERTS_DE_QFLG_REQ_INFO) + receiver = BIF_P->common.id; /* Notify ourselves... */ + /* else: someone else will notify us... */ + } + /* else: still empty queue... */ + } + } + /* else: Already requested... */ + + erts_de_runlock(dep); + + if (is_internal_pid(receiver)) + notify_dist_data(BIF_P, receiver); + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_ctrl_put_data_2(BIF_ALIST_2) +{ + DistEntry *dep; + ErlDrvSizeT size; + Eterm input_handler; + + if (is_binary(BIF_ARG_2)) + size = binary_size(BIF_ARG_2); + else if (is_nil(BIF_ARG_2)) + size = 0; + else if (is_list(BIF_ARG_2)) + BIF_TRAP2(dist_ctrl_put_data_trap, + BIF_P, BIF_ARG_1, BIF_ARG_2); + else + BIF_ERROR(BIF_P, BADARG); + + dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + if (!dep) + BIF_ERROR(BIF_P, BADARG); + + input_handler = (Eterm) erts_atomic_read_nob(&dep->input_handler); + + if (input_handler != BIF_P->common.id) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + erts_atomic64_inc_nob(&dep->in); + + if (size != 0) { + byte *data, *temp_alloc = NULL; + + data = (byte *) erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc); + if (!data) + BIF_ERROR(BIF_P, BADARG); + + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + + (void) erts_net_message(NULL, dep, NULL, 0, data, size); + /* + * We ignore any decode failures. On fatal failures the + * connection will be taken down by killing the + * distribution channel controller... + */ + + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BUMP_REDS(BIF_P, 5); + + erts_free_aligned_binary_bytes(temp_alloc); + + } + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_get_stat_1(BIF_ALIST_1) +{ + Sint64 read, write, pend; + Eterm res, *hp, **hpp; + Uint sz, *szp; + DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1); + + if (!dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + read = (Sint64) erts_atomic64_read_nob(&dep->in); + write = (Sint64) erts_atomic64_read_nob(&dep->out); + pend = (Sint64) erts_atomic_read_nob(&dep->qsize); + + erts_de_runlock(dep); + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + res = erts_bld_tuple(hpp, szp, 4, + am_ok, + erts_bld_sint64(hpp, szp, read), + erts_bld_sint64(hpp, szp, write), + pend ? am_true : am_false); + if (hpp) + break; + hp = HAlloc(BIF_P, sz); + hpp = &hp; + szp = NULL; + } + + BIF_RET(res); +} + +BIF_RETTYPE +dist_ctrl_input_handler_2(BIF_ALIST_2) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + if (is_not_internal_pid(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) BIF_ARG_2); + + BIF_RET(am_ok); +} + +BIF_RETTYPE +dist_ctrl_get_data_1(BIF_ALIST_1) +{ + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); + int reds = 1; + ErtsDistOutputBuf *obuf; + Eterm *hp; + ProcBin *pb; + erts_aint_t qsize; + + if (!dep) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (erts_dhandle_to_dist_entry(BIF_ARG_1) != dep) + BIF_ERROR(BIF_P, BADARG); + + erts_de_rlock(dep); + + if (dep->status & ERTS_DE_SFLG_EXITING) + goto return_none; + + ASSERT(dep->cid == BIF_P->common.id); + +#if 0 + if (dep->finalized_out_queue.first) { + obuf = dep->finalized_out_queue.first; + dep->finalized_out_queue.first = obuf->next; + if (!obuf->next) + dep->finalized_out_queue.last = NULL; + } + else +#endif + { + if (!dep->tmp_out_queue.first) { + ASSERT(!dep->tmp_out_queue.last); + qsize = erts_atomic_read_acqb(&dep->qsize); + if (qsize > 0) { + erts_mtx_lock(&dep->qlock); + dep->tmp_out_queue.first = dep->out_queue.first; + dep->tmp_out_queue.last = dep->out_queue.last; + dep->out_queue.first = NULL; + dep->out_queue.last = NULL; + erts_mtx_unlock(&dep->qlock); + } + } + + if (!dep->tmp_out_queue.first) { + ASSERT(!dep->tmp_out_queue.last); + return_none: + erts_de_runlock(dep); + BIF_RET(am_none); + } + else { + obuf = dep->tmp_out_queue.first; + dep->tmp_out_queue.first = obuf->next; + if (!obuf->next) + dep->tmp_out_queue.last = NULL; + } + + obuf->extp = erts_encode_ext_dist_header_finalize(obuf->extp, + dep->cache, + dep->flags); + reds += ERTS_PORT_REDS_DIST_CMD_FINALIZE; + if (!(dep->flags & DFLAG_DIST_HDR_ATOM_CACHE)) + *--obuf->extp = PASS_THROUGH; /* 'pass through' needed */ + ASSERT(&obuf->data[0] <= obuf->extp + && obuf->extp < obuf->ext_endp); + } + + erts_atomic64_inc_nob(&dep->out); + + erts_de_runlock(dep); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + pb = (ProcBin *) (char *) hp; + pb->thing_word = HEADER_PROC_BIN; + pb->size = obuf->ext_endp - obuf->extp; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = ErtsDistOutputBuf2Binary(obuf); + pb->bytes = (byte*) obuf->extp; + pb->flags = 0; + + qsize = erts_atomic_add_read_nob(&dep->qsize, -size_obuf(obuf)); + ASSERT(qsize >= 0); + + if (qsize < erts_dist_buf_busy_limit/2 + && (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY)) { + ErtsProcList *resume_procs = NULL; + erts_mtx_lock(&dep->qlock); + resume_procs = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); + erts_mtx_unlock(&dep->qlock); + if (resume_procs) { + int resumed = erts_resume_processes(resume_procs); + reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; + } + } + + BIF_RET2(make_binary(pb), reds); +} + void erts_dist_port_not_busy(Port *prt) { @@ -2386,21 +2895,23 @@ erts_dist_port_not_busy(Port *prt) void erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) { - erts_smp_de_rwlock(dep); - if (is_internal_port(dep->cid) - && connection_id == dep->connection_id + erts_de_rwlock(dep); + if (connection_id == dep->connection_id && !(dep->status & ERTS_DE_SFLG_EXITING)) { dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_mtx_lock(&dep->qlock); - ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); - dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_mtx_unlock(&dep->qlock); + erts_mtx_lock(&dep->qlock); + ASSERT(!(erts_atomic32_read_nob(&dep->qflgs) & ERTS_DE_QFLG_EXIT)); + erts_atomic32_read_bor_nob(&dep->qflgs, ERTS_DE_QFLG_EXIT); + erts_mtx_unlock(&dep->qlock); - erts_schedule_dist_command(NULL, dep); + if (is_internal_port(dep->cid)) + erts_schedule_dist_command(NULL, dep); + else if (is_internal_pid(dep->cid)) + schedule_kill_dist_ctrl_proc(dep->cid); } - erts_smp_de_rwunlock(dep); + erts_de_rwunlock(dep); } struct print_to_data { @@ -2515,9 +3026,6 @@ info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connecte } erts_print(to, arg, "Name: %T", dep->sysname); -#ifdef DEBUG - erts_print(to, arg, " (refc=%d)", erts_smp_refc_read(&dep->refc, 0)); -#endif erts_print(to, arg, "\n"); if (!connected && is_nil(dep->cid)) { if (dep->nlinks) { @@ -2637,32 +3145,46 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) goto error; } - net_kernel = erts_whereis_process(BIF_P, ERTS_PROC_LOCK_MAIN, - am_net_kernel, ERTS_PROC_LOCK_MAIN, 0); - if (!net_kernel) + net_kernel = erts_whereis_process(BIF_P, + ERTS_PROC_LOCK_MAIN, + am_net_kernel, + ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS, + 0); + if (!net_kernel || ERTS_PROC_GET_DIST_ENTRY(net_kernel)) goto error; /* By setting F_DISTRIBUTION on net_kernel, - * do_net_exist will be called when net_kernel is terminated !! */ + * erts_do_net_exits will be called when net_kernel is terminated !! */ net_kernel->flags |= F_DISTRIBUTION; - if (net_kernel != BIF_P) - erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(net_kernel, + (ERTS_PROC_LOCK_STATUS + | ((net_kernel != BIF_P) + ? ERTS_PROC_LOCK_MAIN + : 0))); #ifdef DEBUG - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(!erts_visible_dist_entries && !erts_hidden_dist_entries); - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); #endif - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); inc_no_nodes(); erts_set_this_node(BIF_ARG_1, (Uint32) creation); erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + /* + * Note erts_this_dist_entry is changed by erts_set_this_node(), + * so we *need* to use the new one after erts_set_this_node() + * is called. + */ + erts_ref_dist_entry(erts_this_dist_entry); + ERTS_PROC_SET_DIST_ENTRY(net_kernel, erts_this_dist_entry); BIF_RET(am_true); @@ -2693,18 +3215,18 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) Eterm ic, oc; Eterm *tp; DistEntry *dep = NULL; + ErtsProcLocks proc_unlock = 0; + Process *proc; Port *pp = NULL; - /* Prepare for success */ - ERTS_BIF_PREP_RET(ret, am_true); - /* * Check and pick out arguments */ if (!is_node_name_atom(BIF_ARG_1) || - is_not_internal_port(BIF_ARG_2) || - (erts_this_node->sysname == am_Noname)) { + !(is_internal_port(BIF_ARG_2) + || is_internal_pid(BIF_ARG_2)) + || (erts_this_node->sysname == am_Noname)) { goto badarg; } @@ -2748,77 +3270,124 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) else if (!dep) goto system_limit; /* Should never happen!!! */ - pp = erts_id2port_sflgs(BIF_ARG_2, - BIF_P, - ERTS_PROC_LOCK_MAIN, - ERTS_PORT_SFLGS_INVALID_LOOKUP); - erts_smp_de_rwlock(dep); + if (is_internal_pid(BIF_ARG_2)) { + if (BIF_P->common.id == BIF_ARG_2) { + proc_unlock = 0; + proc = BIF_P; + } + else { + proc_unlock = ERTS_PROC_LOCK_MAIN; + proc = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_2, proc_unlock); + } + erts_de_rwlock(dep); - if (!pp || (erts_atomic32_read_nob(&pp->state) - & ERTS_PORT_SFLG_EXITING)) - goto badarg; + if (!proc) + goto badarg; + else if (proc == ERTS_PROC_LOCK_BUSY) { + proc_unlock = 0; + goto yield; + } - if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) - goto badarg; + erts_proc_lock(proc, ERTS_PROC_LOCK_STATUS); + proc_unlock |= ERTS_PROC_LOCK_STATUS; + + if (ERTS_PROC_GET_DIST_ENTRY(proc)) { + if (dep == ERTS_PROC_GET_DIST_ENTRY(proc) + && (proc->flags & F_DISTRIBUTION) + && dep->cid == BIF_ARG_2) { + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + goto done; + } + goto badarg; + } + + if (is_not_nil(dep->cid)) + goto badarg; + + proc->flags |= F_DISTRIBUTION; + ERTS_PROC_SET_DIST_ENTRY(proc, dep); - if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) - goto done; /* Already set */ + proc_unlock &= ~ERTS_PROC_LOCK_STATUS; + erts_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); + + dep->send = NULL; /* Only for distr ports... */ - if (dep->status & ERTS_DE_SFLG_EXITING) { - /* Suspend on dist entry waiting for the exit to finish */ - ErtsProcList *plp = erts_proclist_create(BIF_P); - plp->next = NULL; - erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_mtx_lock(&dep->qlock); - erts_proclist_store_last(&dep->suspended, plp); - erts_smp_mtx_unlock(&dep->qlock); - goto yield; } + else { - ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING)); + pp = erts_id2port_sflgs(BIF_ARG_2, + BIF_P, + ERTS_PROC_LOCK_MAIN, + ERTS_PORT_SFLGS_INVALID_LOOKUP); + erts_de_rwlock(dep); - if (pp->dist_entry || is_not_nil(dep->cid)) - goto badarg; + if (!pp || (erts_atomic32_read_nob(&pp->state) + & ERTS_PORT_SFLG_EXITING)) + goto badarg; - erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + if ((pp->drv_ptr->flags & ERL_DRV_FLAG_SOFT_BUSY) == 0) + goto badarg; - /* - * Dist-ports do not use the "busy port message queue" functionality, but - * instead use "busy dist entry" functionality. - */ - { - ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; - erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); - } + if (dep->cid == BIF_ARG_2 && pp->dist_entry == dep) { + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + goto done; /* Already set */ + } - pp->dist_entry = dep; + if (dep->status & ERTS_DE_SFLG_EXITING) { + /* Suspend on dist entry waiting for the exit to finish */ + ErtsProcList *plp = erts_proclist_create(BIF_P); + plp->next = NULL; + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + erts_mtx_lock(&dep->qlock); + erts_proclist_store_last(&dep->suspended, plp); + erts_mtx_unlock(&dep->qlock); + goto yield; + } - dep->version = version; - dep->creation = 0; + ASSERT(!(dep->status & ERTS_DE_SFLG_EXITING)); - ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); + if (pp->dist_entry || is_not_nil(dep->cid)) + goto badarg; -#if 1 - dep->send = (pp->drv_ptr->outputv - ? dist_port_commandv - : dist_port_command); -#else - dep->send = dist_port_command; -#endif - ASSERT(dep->send); + erts_atomic32_read_bor_nob(&pp->state, ERTS_PORT_SFLG_DISTRIBUTION); + + pp->dist_entry = dep; + + ASSERT(pp->drv_ptr->outputv || pp->drv_ptr->output); + + dep->send = (pp->drv_ptr->outputv + ? dist_port_commandv + : dist_port_command); + ASSERT(dep->send); + + /* + * Dist-ports do not use the "busy port message queue" functionality, but + * instead use "busy dist entry" functionality. + */ + { + ErlDrvSizeT disable = ERL_DRV_BUSY_MSGQ_DISABLED; + erl_drv_busy_msgq_limits(ERTS_Port2ErlDrvPort(pp), &disable, NULL); + } + + } + + dep->version = version; + dep->creation = 0; #ifdef DEBUG - erts_smp_mtx_lock(&dep->qlock); - ASSERT(dep->qsize == 0); - erts_smp_mtx_unlock(&dep->qlock); + ASSERT(erts_atomic_read_nob(&dep->qsize) == 0); #endif - erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); - if (flags & DFLAG_DIST_HDR_ATOM_CACHE) create_cache(dep); - erts_smp_de_rwunlock(dep); + erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); + + erts_de_rwunlock(dep); + + ERTS_BIF_PREP_RET(ret, erts_make_dhandle(BIF_P, dep)); + dep = NULL; /* inc of refc transferred to port (dist_entry field) */ inc_no_nodes(); @@ -2831,13 +3400,16 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) done: if (dep && dep != erts_this_dist_entry) { - erts_smp_de_rwunlock(dep); + erts_de_rwunlock(dep); erts_deref_dist_entry(dep); } if (pp) erts_port_release(pp); + if (proc_unlock) + erts_proc_unlock(proc, proc_unlock); + return ret; yield: @@ -2883,7 +3455,7 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) if (BIF_P->common.id == local) { lp_locks = ERTS_PROC_LOCKS_ALL; lp = BIF_P; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_ALL_MINOR); } else { lp_locks = ERTS_PROC_LOCKS_XSIG_SEND; @@ -2902,21 +3474,17 @@ BIF_RETTYPE dist_exit_3(BIF_ALIST_3) NIL, NULL, 0); -#ifdef ERTS_SMP if (lp == BIF_P) lp_locks &= ~ERTS_PROC_LOCK_MAIN; -#endif - erts_smp_proc_unlock(lp, lp_locks); + erts_proc_unlock(lp, lp_locks); if (lp == BIF_P) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&BIF_P->state); + erts_aint32_t state = erts_atomic32_read_acqb(&BIF_P->state); /* * We may have exited current process and may have to take action. */ if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { -#ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); -#endif ERTS_BIF_EXITED(BIF_P); } } @@ -3002,7 +3570,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) length = 0; - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(erts_no_of_not_connected_dist_entries > 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); @@ -3019,7 +3587,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) result = NIL; if (length == 0) { - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); goto done; } @@ -3050,7 +3618,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) hp += 2; } ASSERT(endp == hp); - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); done: UnUseTmpHeap(2,BIF_P); @@ -3105,15 +3673,15 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) if (dep == erts_this_dist_entry) goto done; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK); - erts_smp_de_rlock(dep); + erts_proc_lock(p, ERTS_PROC_LOCK_LINK); + erts_de_rlock(dep); if (ERTS_DE_IS_NOT_CONNECTED(dep)) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); - erts_smp_de_runlock(dep); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_de_runlock(dep); goto do_trap; } - erts_smp_de_links_lock(dep); - erts_smp_de_runlock(dep); + erts_de_links_lock(dep); + erts_de_runlock(dep); if (Bool == am_true) { ASSERT(dep->cid != NIL); @@ -3140,11 +3708,10 @@ monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options) } } - erts_smp_de_links_unlock(dep); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_de_links_unlock(dep); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); done: - erts_deref_dist_entry(dep); BIF_RET(am_true); } @@ -3173,9 +3740,9 @@ BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1) if (de == erts_this_dist_entry) { BIF_RET(am_true); } - erts_smp_de_rlock(de); + erts_de_rlock(de); f = de->flags; - erts_smp_de_runlock(de); + erts_de_runlock(de); BIF_RET(((f & DFLAG_UNICODE_IO) ? am_true : am_false)); } @@ -3205,7 +3772,7 @@ struct ErtsNodesMonitor_ { Uint16 no; }; -static erts_smp_mtx_t nodes_monitors_mtx; +static erts_mtx_t nodes_monitors_mtx; static ErtsNodesMonitor *nodes_monitors; static ErtsNodesMonitor *nodes_monitors_end; @@ -3223,7 +3790,8 @@ static ErtsNodesMonitor *nodes_monitors_end; static void init_nodes_monitors(void) { - erts_smp_mtx_init(&nodes_monitors_mtx, "nodes_monitors"); + erts_mtx_init(&nodes_monitors_mtx, "nodes_monitors", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); nodes_monitors = NULL; nodes_monitors_end = NULL; } @@ -3348,10 +3916,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas } #endif - ERTS_SMP_LC_ASSERT(!c_p + ERTS_LC_ASSERT(!c_p || (erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN)); - erts_smp_mtx_lock(&nodes_monitors_mtx); + erts_mtx_lock(&nodes_monitors_mtx); for (nmp = nodes_monitors; nmp; nmp = nmp->next) { int i; @@ -3384,7 +3952,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas if (rp) { if (rp == c_p) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } rp = nmp->proc; @@ -3411,10 +3979,10 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas if (rp) { if (rp == c_p) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } - erts_smp_mtx_unlock(&nodes_monitors_mtx); + erts_mtx_unlock(&nodes_monitors_mtx); } static Eterm @@ -3424,8 +3992,8 @@ insert_nodes_monitor(Process *c_p, Uint32 opts) Eterm res = am_false; ErtsNodesMonitor *xnmp, *nmp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&nodes_monitors_mtx)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx)); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); xnmp = c_p->nodes_monitors; if (xnmp) { @@ -3509,8 +4077,8 @@ remove_nodes_monitors(Process *c_p, Uint32 opts, int all) Eterm res = am_false; ErtsNodesMonitor *nmp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&nodes_monitors_mtx)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&nodes_monitors_mtx)); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); nmp = c_p->nodes_monitors; ASSERT(!nmp || !nmp->prev || nmp->prev->proc != c_p); @@ -3552,23 +4120,23 @@ remove_nodes_monitors(Process *c_p, Uint32 opts, int all) void erts_delete_nodes_monitors(Process *c_p, ErtsProcLocks locks) { -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_CHECK) if (c_p) { ErtsProcLocks might_unlock = locks & ~ERTS_PROC_LOCK_MAIN; if (might_unlock) erts_proc_lc_might_unlock(c_p, might_unlock); } #endif - if (erts_smp_mtx_trylock(&nodes_monitors_mtx) == EBUSY) { + if (erts_mtx_trylock(&nodes_monitors_mtx) == EBUSY) { ErtsProcLocks unlock_locks = locks & ~ERTS_PROC_LOCK_MAIN; if (c_p && unlock_locks) - erts_smp_proc_unlock(c_p, unlock_locks); - erts_smp_mtx_lock(&nodes_monitors_mtx); + erts_proc_unlock(c_p, unlock_locks); + erts_mtx_lock(&nodes_monitors_mtx); if (c_p && unlock_locks) - erts_smp_proc_lock(c_p, unlock_locks); + erts_proc_lock(c_p, unlock_locks); } remove_nodes_monitors(c_p, 0, 1); - erts_smp_mtx_unlock(&nodes_monitors_mtx); + erts_mtx_unlock(&nodes_monitors_mtx); } Eterm @@ -3579,7 +4147,7 @@ erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist) Uint16 opts = (Uint16) 0; ASSERT(c_p); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); if (on != am_true && on != am_false) return THE_NON_VALUE; @@ -3635,14 +4203,14 @@ erts_monitor_nodes(Process *c_p, Eterm on, Eterm olist) return THE_NON_VALUE; } - erts_smp_mtx_lock(&nodes_monitors_mtx); + erts_mtx_lock(&nodes_monitors_mtx); if (on == am_true) res = insert_nodes_monitor(c_p, opts); else res = remove_nodes_monitors(c_p, opts, 0); - erts_smp_mtx_unlock(&nodes_monitors_mtx); + erts_mtx_unlock(&nodes_monitors_mtx); return res; } @@ -3665,8 +4233,8 @@ erts_processes_monitoring_nodes(Process *c_p) #endif ASSERT(c_p); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); - erts_smp_mtx_lock(&nodes_monitors_mtx); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + erts_mtx_lock(&nodes_monitors_mtx); sz = 0; szp = &sz; @@ -3715,7 +4283,7 @@ erts_processes_monitoring_nodes(Process *c_p) ASSERT(hp == hend); - erts_smp_mtx_unlock(&nodes_monitors_mtx); + erts_mtx_unlock(&nodes_monitors_mtx); return res; } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 3e17645997..d4765c50b8 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -44,6 +44,7 @@ #define DFLAG_UTF8_ATOMS 0x10000 #define DFLAG_MAP_TAG 0x20000 #define DFLAG_BIG_CREATION 0x40000 +#define DFLAG_SEND_SENDER 0x80000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -74,6 +75,9 @@ #define DOP_DEMONITOR_P 20 #define DOP_MONITOR_P_EXIT 21 +#define DOP_SEND_SENDER 22 +#define DOP_SEND_SENDER_TT 23 + /* distribution trap functions */ extern Export* dsend2_trap; extern Export* dsend3_trap; @@ -100,7 +104,7 @@ typedef struct { } ErtsDSigData; #define ERTS_DE_IS_NOT_CONNECTED(DEP) \ - (ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \ + (ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \ || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \ (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING))) @@ -153,21 +157,18 @@ erts_dsig_prepare(ErtsDSigData *dsdp, if (!dep) return ERTS_DSIG_PREP_NOT_CONNECTED; if (dspl == ERTS_DSP_RWLOCK) - erts_smp_de_rwlock(dep); + erts_de_rwlock(dep); else - erts_smp_de_rlock(dep); + erts_de_rlock(dep); if (ERTS_DE_IS_NOT_CONNECTED(dep)) { failure = ERTS_DSIG_PREP_NOT_CONNECTED; goto fail; } if (no_suspend) { - failure = ERTS_DSIG_PREP_CONNECTED; - erts_smp_mtx_lock(&dep->qlock); - if (dep->qflgs & ERTS_DE_QFLG_BUSY) + if (erts_atomic32_read_acqb(&dep->qflgs) & ERTS_DE_QFLG_BUSY) { failure = ERTS_DSIG_PREP_WOULD_SUSPEND; - erts_smp_mtx_unlock(&dep->qlock); - if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND) goto fail; + } } dsdp->proc = proc; dsdp->dep = dep; @@ -175,14 +176,14 @@ erts_dsig_prepare(ErtsDSigData *dsdp, dsdp->connection_id = dep->connection_id; dsdp->no_suspend = no_suspend; if (dspl == ERTS_DSP_NO_LOCK) - erts_smp_de_runlock(dep); + erts_de_runlock(dep); return ERTS_DSIG_PREP_CONNECTED; fail: if (dspl == ERTS_DSP_RWLOCK) - erts_smp_de_rwunlock(dep); + erts_de_rwunlock(dep); else - erts_smp_de_runlock(dep); + erts_de_runlock(dep); return failure; } @@ -194,7 +195,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) Eterm id; if (prt) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT((erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_DEAD) == 0); ASSERT(prt->dist_entry); @@ -204,7 +205,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) } else { ASSERT(dist_entry); - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx) || erts_lc_rwmtx_is_rwlocked(&dist_entry->rwmtx)); ASSERT(is_internal_port(dist_entry->cid)); @@ -212,7 +213,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry) id = dep->cid; } - if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) + if (!erts_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD); } @@ -238,7 +239,7 @@ erts_remove_dist_link(ErtsDistLinkData *dldp, Eterm rid, DistEntry *dep) { - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); dldp->d_lnk = erts_lookup_link(dep->nlinks, lid); if (!dldp->d_lnk) dldp->d_sub_lnk = NULL; @@ -248,7 +249,7 @@ erts_remove_dist_link(ErtsDistLinkData *dldp, ? NULL : erts_remove_link(&dep->nlinks, lid)); } - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); } ERTS_GLB_INLINE int @@ -349,6 +350,7 @@ typedef struct { Eterm ctl_heap[6]; ErtsDSigData dsd; DistEntry* dep_to_deref; + DistEntry *dep; struct erts_dsig_send_context dss; Eterm return_term; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 169e1e423d..88285d8be6 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -83,14 +83,6 @@ #define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC #define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC -#ifndef ERTS_SMP -# undef ERTS_ALC_DEFAULT_ACUL -# define ERTS_ALC_DEFAULT_ACUL 0 -# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC -# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 -# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC -# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 -#endif #ifdef DEBUG static Uint install_debug_functions(void); @@ -148,7 +140,7 @@ enum { }; typedef struct { - erts_smp_atomic32_t refc; + erts_atomic32_t refc; int only_sz; int internal; Uint req_sched; @@ -393,6 +385,7 @@ set_default_temp_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; + ip->disable_allowed = 0; ip->carrier_migration_allowed = 0; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; @@ -528,7 +521,6 @@ set_default_test_alloc_opts(struct au_init *ip) } -#ifdef ERTS_SMP static void adjust_tpref(struct au_init *ip, int no_sched) @@ -551,7 +543,6 @@ adjust_tpref(struct au_init *ip, int no_sched) } } -#endif static void handle_args(int *, char **, erts_alc_hndl_args_init_t *); @@ -580,7 +571,6 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size) if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) { int j; -#ifdef ERTS_SMP if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) { int i; ErtsAllocatorThrSpec_t* tspec; @@ -596,7 +586,6 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size) } } else -#endif { Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra; for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { @@ -619,7 +608,6 @@ strategy_support_carrier_migration(struct au_init *auip) static ERTS_INLINE void adjust_carrier_migration_support(struct au_init *auip) { -#ifdef ERTS_SMP if (auip->init.util.acul) { auip->thr_spec = -1; /* Need thread preferred */ @@ -633,9 +621,6 @@ adjust_carrier_migration_support(struct au_init *auip) auip->init.aoff.flavor = AOFF_BF; } } -#else - auip->init.util.acul = 0; -#endif } void @@ -660,18 +645,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] = ERTS_LINK_SH_SIZE * sizeof(Uint); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] - = sizeof(ErtsDrvEventDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)] = sizeof(ErtsDrvSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_SEL_D_STATE)] = sizeof(ErtsNifSelectDataState); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)] = sizeof(ErtsMessageRef); -#ifdef ERTS_SMP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)] = sizeof(ErtsThrQElement_t); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_LL_PTIMER)] = erts_timer_type_size(ERTS_ALC_T_LL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_HL_PTIMER)] @@ -734,20 +715,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #endif } -#ifndef ERTS_SMP - init.sl_alloc.thr_spec = 0; - init.std_alloc.thr_spec = 0; - init.ll_alloc.thr_spec = 0; - init.eheap_alloc.thr_spec = 0; - init.binary_alloc.thr_spec = 0; - init.ets_alloc.thr_spec = 0; - init.driver_alloc.thr_spec = 0; - init.fix_alloc.thr_spec = 0; - init.literal_alloc.thr_spec = 0; -#ifdef ERTS_ALC_A_EXEC - init.exec_alloc.thr_spec = 0; -#endif -#endif /* Make adjustments for carrier migration support */ init.temp_alloc.init.util.acul = 0; @@ -798,7 +765,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #endif } -#ifdef ERTS_SMP /* Only temp_alloc can use thread specific interface */ if (init.temp_alloc.thr_spec) init.temp_alloc.thr_spec = erts_no_schedulers; @@ -817,10 +783,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) adjust_tpref(&init.exec_alloc, erts_no_schedulers); #endif -#else - /* No thread specific if not smp */ - init.temp_alloc.thr_spec = 0; -#endif /* * The following allocators cannot be run with afit strategy. @@ -839,10 +801,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) refuse_af_strategy(&init.exec_alloc); #endif -#ifdef ERTS_SMP if (!init.temp_alloc.thr_spec) refuse_af_strategy(&init.temp_alloc); -#endif erts_mtrace_pre_init(); #if HAVE_ERTS_MSEG @@ -1006,8 +966,6 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) return; } -#ifdef USE_THREADS -#ifdef ERTS_SMP if (init->thr_spec) { if (init->thr_spec > 0) { af->alloc = erts_alcu_alloc_thr_spec; @@ -1037,7 +995,6 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) ai->thr_spec = tspec->size; } else -#endif if (init->init.util.ts) { af->alloc = erts_alcu_alloc_ts; if (init->init.util.fix_type_size) @@ -1049,21 +1006,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) af->free = erts_alcu_free_ts; } else -#endif { -#ifdef ERTS_SMP erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n", init->init.util.name_prefix); -#else - af->alloc = erts_alcu_alloc; - if (init->init.util.fix_type_size) - af->realloc = erts_realloc_fixed_size; - else if (init->init.util.ramv) - af->realloc = erts_alcu_realloc_mv; - else - af->realloc = erts_alcu_realloc; - af->free = erts_alcu_free; -#endif } af->extra = NULL; ai->alloc_util = 1; @@ -1548,8 +1493,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) &init->ll_alloc, &init->driver_alloc, &init->fix_alloc, - &init->sl_alloc, - &init->temp_alloc + &init->sl_alloc /* test_alloc not affected by +Mea??? or +Mu??? */ }; int aui_sz = (int) sizeof(aui)/sizeof(aui[0]); @@ -1895,9 +1839,7 @@ erts_alloc_register_scheduler(void *vesdp) int ix = (int) esdp->no; int aix; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) { ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix]; esdp->alloc_data.deallctr[aix] = NULL; @@ -1915,7 +1857,6 @@ erts_alloc_register_scheduler(void *vesdp) } } -#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, @@ -1944,12 +1885,10 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, } } } -#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) { -#ifdef ERTS_SMP ErtsAllocatorThrSpec_t *tspec; tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled) @@ -1957,11 +1896,6 @@ erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) return erts_alcu_fix_alloc_shrink( erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); -#else - if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra) - return erts_alcu_fix_alloc_shrink( - erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs); -#endif return 0; } @@ -2165,7 +2099,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) int only_one_value = 0; ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}}; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); /* Figure out whats wanted... */ @@ -2338,10 +2272,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (proc) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(proc)); /* We'll need locks early in the lock order */ - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); } /* Calculate values needed... */ @@ -2499,7 +2433,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) Uint *hp; Uint hsz; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); if (only_one_value) { ASSERT(length == 1); @@ -2548,11 +2482,11 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) Uint reserved_atom_space, atom_space; if (proc) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(proc)); /* We'll need locks early in the lock order */ - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); } i = 0; @@ -2704,7 +2638,7 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) Uint hsz; Uint *hszp; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); hpp = NULL; hsz = 0; @@ -2792,7 +2726,7 @@ erts_allocator_info(fmtfn_t to, void *arg) { ErtsAlcType_t a; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) { int ai; @@ -2845,11 +2779,7 @@ erts_allocator_info(fmtfn_t to, void *arg) #if HAVE_ERTS_MSEG { struct erts_mmap_info_struct emis; -#ifdef ERTS_SMP int max = (int) erts_no_schedulers; -#else - int max = 0; -#endif int i; for (i = 0; i <= max; i++) { erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i); @@ -3310,7 +3240,7 @@ reply_alloc_info(void *vair) case ERTS_ALC_INFO_A_DISABLED_EXEC: break; case ERTS_ALC_INFO_A_MSEG_ALLOC: -#if HAVE_ERTS_MSEG && defined(ERTS_SMP) +#if HAVE_ERTS_MSEG alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc"); ainfo = erts_mseg_info(sched_id, NULL, NULL, hpp != NULL, air->only_sz, hpp, szp); @@ -3364,10 +3294,10 @@ reply_alloc_info(void *vair) if (air->req_sched == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); - if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0) { + if (erts_atomic32_dec_read_nob(&air->refc) == 0) { erts_iref_storage_clean(&air->iref); aireq_free(air); } @@ -3446,18 +3376,16 @@ erts_request_alloc_info(struct process *c_p, air->allocs[airix] = ERTS_ALC_A_INVALID; - erts_smp_atomic32_init_nob(&air->refc, + erts_atomic32_init_nob(&air->refc, (erts_aint32_t) erts_no_schedulers); erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, reply_alloc_info, (void *) air); -#endif reply_alloc_info((void *) air); @@ -3532,35 +3460,29 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf: switch (op) { case 0xf00: -#ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); else -#endif return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF, (void *) a1, (Uint) a2); case 0xf01: -#ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); else -#endif return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2, (Uint) a3); case 0xf02: -#ifdef USE_THREADS if (((Allctr_t *) a1)->thread_safe) erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); else -#endif erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); return 0; case 0xf03: { @@ -3571,11 +3493,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) init.enable = 1; init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; -#ifdef ERTS_SMP init.init.util.ts = 1; -#else - init.init.util.ts = a2 ? 1 : 0; -#endif if ((char **) a3) { char **argv = (char **) a3; int i = 0; @@ -3630,7 +3548,6 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) erts_alcu_stop((Allctr_t *) a1); erts_free(ERTS_ALC_T_UNDEF, (void *) a1); break; -#ifdef USE_THREADS case 0xf05: return (UWord) 1; case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe; #ifdef ETHR_NO_FORKSAFETY @@ -3700,12 +3617,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_thr_exit((void *) a1); ERTS_ALC_TEST_ABORT; break; -#endif /* #ifdef USE_THREADS */ -#ifdef ERTS_SMP case 0xf13: return (UWord) 1; -#else - case 0xf13: return (UWord) 0; -#endif case 0xf14: return (UWord) erts_alloc(ERTS_ALC_T_TEST, (Uint)a1); case 0xf15: erts_free(ERTS_ALC_T_TEST, (void*)a1); return 0; @@ -3819,7 +3731,8 @@ hdbg_init(void) hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; free_hdbg_mblks = &hdbg_mblks[0]; used_hdbg_mblks = NULL; - erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); } static void *check_memory_fence(void *ptr, @@ -3908,10 +3821,8 @@ void check_allocators(void) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra; Allctr_t *allctr = real_af->extra; Carrier_t *ct; -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); -#endif if (allctr->check_mbc) { for (ct = allctr->mbc_list.first; ct; ct = ct->next) { @@ -3919,10 +3830,8 @@ void check_allocators(void) allctr->check_mbc(allctr,ct); } } -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); -#endif } } } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 7b5cbe2178..117f96a4ad 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -27,9 +27,7 @@ #include "erl_thr_progress.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_alloc_util.h" -#ifdef USE_THREADS #include "erl_threads.h" -#endif #include "erl_mmap.h" #ifdef DEBUG @@ -154,12 +152,10 @@ void erts_allctr_wrapper_pre_lock(void); void erts_allctr_wrapper_pre_unlock(void); void erts_alloc_register_scheduler(void *vesdp); -#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, ErtsThrPrgrVal *thr_prgr_p, int *more_work); -#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) @@ -338,54 +334,23 @@ erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) #define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ -ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, \ - (void) 0, (void) 0, (void) 0) - -#define ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ -static erts_smp_spinlock_t NAME##_lck; \ -ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, \ - erts_smp_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\ - erts_smp_spin_lock(&NAME##_lck), \ - erts_smp_spin_unlock(&NAME##_lck)) - -#ifdef ERTS_SMP + ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, (void) 0, (void) 0, (void) 0) #define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ -ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) - -#else /* !ERTS_SMP */ - -#define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ -static erts_mtx_t NAME##_lck; \ -ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, \ - erts_mtx_init(NAME##_lck, #NAME "_alloc_lock"), \ - erts_mtx_lock(&NAME##_lck), \ - erts_mtx_unlock(&NAME##_lck)) - - -#endif - -#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ) \ -ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0) +ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT) #define ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ) \ static erts_spinlock_t NAME##_lck; \ ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, \ - erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\ + erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock", NIL, \ + ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR),\ erts_spin_lock(&NAME##_lck), \ erts_spin_unlock(&NAME##_lck)) -#ifdef ERTS_SMP -#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ) \ +#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ) \ ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ) -#else /* !ERTS_SMP */ - -#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ) \ - ERTS_PALLOC_IMPL(NAME, TYPE, PASZ) - -#endif #define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK) \ ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, ILCK, LCK, ULCK) \ @@ -409,21 +374,11 @@ NAME##_free(TYPE *p) \ erts_free(ALCT, (void *) p); \ } -#ifdef ERTS_SMP #define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ) \ ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) -#else -#define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ) \ - ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0) -#endif -#ifdef ERTS_SMP #define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \ ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ) -#else -#define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \ -ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, (void) 0, (void) 0, (void) 0) -#endif #define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ) \ @@ -447,8 +402,34 @@ NAME##_free(TYPE *p) \ erts_free(ALCT, (void *) p); \ } +#define ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \ +ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ) + +#define ERTS_THR_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT) \ +ERTS_THR_PREF_AUX(NAME, TYPE, PASZ) \ +static void \ +init_##NAME##_alloc(int nthreads) \ +{ \ + init_##NAME##_pre_alloc(nthreads); \ +} \ +static ERTS_INLINE TYPE * \ +NAME##_alloc(void) \ +{ \ + TYPE *res = NAME##_pre_alloc(); \ + if (!res) \ + res = erts_alloc(ALCT, sizeof(TYPE)); \ + return res; \ +} \ +static ERTS_INLINE void \ +NAME##_free(TYPE *p) \ +{ \ + if (!NAME##_pre_free(p)) \ + erts_free(ALCT, (void *) p); \ +} + + #ifdef DEBUG -#define ERTS_PRE_ALLOC_SIZE(SZ) 2 +#define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) < 1000 ? (SZ)/10 + 10 : 100) #define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T)) #else #define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1) @@ -527,7 +508,8 @@ init_##NAME##_alloc(void) \ { \ sspa_data_##NAME##__ = \ erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ - ERTS_PRE_ALLOC_SIZE((PASZ))); \ + ERTS_PRE_ALLOC_SIZE((PASZ)), \ + 0, NULL); \ } \ \ static TYPE * \ @@ -549,6 +531,57 @@ NAME##_free(TYPE *p) \ (char *) p); \ } + +#define ERTS_THR_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ) \ +union erts_sspa_##NAME##__ { \ + erts_sspa_blk_t next; \ + TYPE type; \ +}; \ + \ +static erts_sspa_data_t *sspa_data_##NAME##__; \ + \ +static void \ +init_##NAME##_alloc(int nthreads) \ +{ \ + sspa_data_##NAME##__ = \ + erts_sspa_create(sizeof(union erts_sspa_##NAME##__), \ + ERTS_PRE_ALLOC_SIZE((PASZ)), \ + nthreads, \ + #NAME); \ +} \ + \ +void \ +erts_##NAME##_alloc_init_thread(void) \ +{ \ + int id = erts_atomic_inc_read_nob(&sspa_data_##NAME##__->id_generator);\ + if (id > sspa_data_##NAME##__->nthreads) { \ + erts_exit(ERTS_ABORT_EXIT, \ + "%s:%d:%s(): Too many threads for '" #NAME "'\n", \ + __FILE__, __LINE__, __func__); \ + } \ + erts_tsd_set(sspa_data_##NAME##__->tsd_key, (void*)(SWord)id); \ +} \ + \ +static TYPE * \ +NAME##_alloc(void) \ +{ \ + int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \ + if (id == 0) \ + return NULL; \ + return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__, \ + id-1); \ +} \ + \ +static int \ +NAME##_free(TYPE *p) \ +{ \ + int id = (int)(SWord)erts_tsd_get(sspa_data_##NAME##__->tsd_key); \ + return erts_sspa_free(sspa_data_##NAME##__, \ + id - 1, \ + (char *) p); \ +} + + #ifdef DEBUG #define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2)) #endif /* #ifdef DEBUG */ diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 8a23a1526e..bf5487d31e 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -53,15 +53,6 @@ # # IMPORTANT! Only use 7-bit ascii text in this file! -+if smp -+disable threads_no_smp -+else -+if threads -+enable threads_no_smp -+else -+disable threads_no_smp -+endif -+endif # --- Allocator declarations ------------------------------------------------- # @@ -77,8 +68,6 @@ allocator SYSTEM true sys_alloc -+if smp - allocator TEMPORARY true temp_alloc allocator SHORT_LIVED true sl_alloc allocator STANDARD true std_alloc @@ -91,22 +80,6 @@ allocator LITERAL true literal_alloc allocator EXEC true exec_alloc +endif -+else # Non smp build - -allocator TEMPORARY false temp_alloc -allocator SHORT_LIVED false sl_alloc -allocator STANDARD false std_alloc -allocator LONG_LIVED false ll_alloc -allocator EHEAP false eheap_alloc -allocator ETS false ets_alloc -allocator FIXED_SIZE false fix_alloc -allocator LITERAL false literal_alloc -+if exec_alloc -allocator EXEC false exec_alloc -+endif - -+endif - allocator BINARY true binary_alloc allocator DRIVER true driver_alloc @@ -285,32 +258,18 @@ type MREF_ENT STANDARD SYSTEM magic_ref_entry type MREF_TAB_BKTS STANDARD SYSTEM magic_ref_table_buckets type MREF_TAB LONG_LIVED SYSTEM magic_ref_table type MINDIRECTION FIXED_SIZE SYSTEM magic_indirection +type BINARY_FIND SHORT_LIVED PROCESSES binary_find -+if threads_no_smp -# Need thread safe allocs, but std_alloc and fix_alloc are not; -# use driver_alloc which is... -type THR_Q_EL DRIVER SYSTEM thr_q_element -type THR_Q_EL_SL DRIVER SYSTEM sl_thr_q_element -type MISC_AUX_WORK DRIVER SYSTEM misc_aux_work -+else type THR_Q_EL STANDARD SYSTEM thr_q_element type THR_Q_EL_SL FIXED_SIZE SYSTEM sl_thr_q_element type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work -+endif type THR_Q STANDARD SYSTEM thr_queue type THR_Q_SL SHORT_LIVED SYSTEM short_lived_thr_queue type THR_Q_LL LONG_LIVED SYSTEM long_lived_thr_queue -+if smp type ASYNC SHORT_LIVED SYSTEM async type ZLIB STANDARD SYSTEM zlib -+else -# sl/std_alloc is not thread safe in non smp build; therefore, we use driver_alloc -type ZLIB DRIVER SYSTEM zlib -type ASYNC DRIVER SYSTEM async -+endif -+if smp type PORT_LOCK STANDARD SYSTEM port_lock type DRIVER_LOCK STANDARD SYSTEM driver_lock type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list @@ -320,33 +279,19 @@ type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data type RELEASE_LAREA SHORT_LIVED SYSTEM release_literal_area -+endif # # Types used for special emulators # -+if threads - type ETHR_STD STANDARD SYSTEM ethread_standard type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived -+endif - -+if shared_heap - -type STACK STANDARD PROCESSES stack -type ACTIVE_PROCS STANDARD PROCESSES active_procs - -+endif - -+if smp type SYS_MSG_Q SHORT_LIVED PROCESSES system_messages_queue type FP_EXCEPTION LONG_LIVED SYSTEM fp_exception type LL_MPATHS LONG_LIVED SYSTEM ll_migration_paths type SL_MPATHS SHORT_LIVED SYSTEM sl_migration_paths -+endif +if hipe @@ -360,14 +305,19 @@ type HIPE_EXEC EXEC CODE hipe_code +endif - - +if heap_frag_elim_test type SSB SHORT_LIVED PROCESSES ssb +endif ++if lcnt + +type LCNT_CARRIER STANDARD SYSTEM lcnt_lock_info_carrier +type LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_sample_vector + ++endif + type DEBUG SHORT_LIVED SYSTEM debugging type DDLL_PROCESS STANDARD SYSTEM ddll_processes @@ -401,11 +351,9 @@ type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request type TEMP_TERM TEMPORARY SYSTEM temp_term type DRV_TAB LONG_LIVED SYSTEM drv_tab type DRV_EV_STATE LONG_LIVED SYSTEM driver_event_state -type DRV_EV_D_STATE FIXED_SIZE SYSTEM driver_event_data_state type DRV_SEL_D_STATE FIXED_SIZE SYSTEM driver_select_data_state type NIF_SEL_D_STATE FIXED_SIZE SYSTEM enif_select_data_state type FD_LIST SHORT_LIVED SYSTEM fd_list -type ACTIVE_FD_ARR SHORT_LIVED SYSTEM active_fd_array type POLLSET LONG_LIVED SYSTEM pollset type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds @@ -424,11 +372,7 @@ type PUTENV_STR SYSTEM SYSTEM putenv_string type PRT_REP_EXIT STANDARD SYSTEM port_report_exit type SYS_BLOCKING STANDARD SYSTEM sys_blocking -+if smp type SYS_WRITE_BUF TEMPORARY SYSTEM sys_write_buf -+else -type SYS_WRITE_BUF BINARY SYSTEM sys_write_buf -+endif +endif diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 6fddba4b34..4d4bddb93f 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -305,9 +305,6 @@ MBC after deallocating first block: # define ERTS_ALC_CPOOL_DEBUG #endif -#ifndef ERTS_SMP -# undef ERTS_ALC_CPOOL_DEBUG -#endif #ifdef ERTS_ALC_CPOOL_DEBUG # define ERTS_ALC_CPOOL_ASSERT(A) \ @@ -322,13 +319,8 @@ MBC after deallocating first block: # define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1) #endif -#ifdef ERTS_SMP #define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit) -#else -#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0) -#endif -#ifdef ERTS_SMP #define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000 #define ERTS_ALC_CPOOL_ALLOC_OP_INC 8 @@ -367,28 +359,16 @@ do { \ } \ } while (0) -#else -#define ERTS_ALC_CPOOL_ALLOC_OP(A) -#define ERTS_ALC_CPOOL_REALLOC_OP(A) -#define ERTS_ALC_CPOOL_FREE_OP(A) -#endif #define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) #define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) #define ERTS_CRR_ALCTR_FLG_MASK (ERTS_CRR_ALCTR_FLG_IN_POOL | \ ERTS_CRR_ALCTR_FLG_BUSY) -#ifdef ERTS_SMP #define SBC_HEADER_SIZE \ (UNIT_CEILING(offsetof(Carrier_t, cpool) \ + ABLK_HDR_SZ) \ - ABLK_HDR_SZ) -#else -#define SBC_HEADER_SIZE \ - (UNIT_CEILING(sizeof(Carrier_t) \ - + ABLK_HDR_SZ) \ - - ABLK_HDR_SZ) -#endif #define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size) @@ -402,7 +382,7 @@ do { \ #define SET_CARRIER_HDR(C, Sz, F, AP) \ (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ - erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) + erts_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) @@ -598,15 +578,11 @@ do { \ (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ } while (0) -#ifdef ERTS_SMP #define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ do { \ (CRR)->cpool.blocks++; \ (CRR)->cpool.blocks_size += (BSZ); \ } while (0) -#else -#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */ -#endif #define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ do { \ @@ -626,7 +602,6 @@ stat_cpool_mbc_blk_free(Allctr_t *allctr, Carrier_t **busy_pcrr_pp, UWord blksz) { -#ifdef ERTS_SMP ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); crr->cpool.blocks--; @@ -651,9 +626,6 @@ stat_cpool_mbc_blk_free(Allctr_t *allctr, #endif return 1; -#else - return 0; -#endif } #define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ @@ -689,12 +661,7 @@ do { \ #endif #ifdef DEBUG -#ifdef USE_THREADS -# ifdef ERTS_SMP # define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking()) -# else -# define IS_ACTUALLY_BLOCKING 0 -# endif #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) { \ @@ -703,7 +670,7 @@ do { \ (A)->debug.saved_tid = 1; \ } \ else { \ - ERTS_SMP_LC_ASSERT( \ + ERTS_LC_ASSERT( \ ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ } \ } \ @@ -711,9 +678,6 @@ do { \ #else #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif -#else -#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) -#endif static void make_name_atoms(Allctr_t *allctr); @@ -862,7 +826,7 @@ erts_alcu_literal_32_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) Uint sz = ERTS_SUPERALIGNED_CEILING(*size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); res = erts_alcu_mseg_alloc(allctr, &sz, flags); if (res) { @@ -880,7 +844,7 @@ erts_alcu_literal_32_mseg_realloc(Allctr_t *allctr, void *seg, Uint new_sz = ERTS_SUPERALIGNED_CEILING(*new_size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); if (seg && old_size) clear_literal_range(seg, old_size); @@ -898,7 +862,7 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, { ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); erts_alcu_mseg_dealloc(allctr, seg, size, flags); @@ -1058,7 +1022,7 @@ erts_alcu_literal_32_sys_alloc(Allctr_t *allctr, Uint* size_p, int superalign) Uint size = ERTS_SUPERALIGNED_CEILING(*size_p); ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); res = erts_alcu_sys_alloc(allctr, &size, 1); if (res) { @@ -1076,7 +1040,7 @@ erts_alcu_literal_32_sys_realloc(Allctr_t *allctr, void *ptr, Uint* size_p, Uint ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); if (ptr && old_size) clear_literal_range(ptr, old_size); @@ -1093,7 +1057,7 @@ erts_alcu_literal_32_sys_dealloc(Allctr_t *allctr, void *ptr, Uint size, int sup { ERTS_LC_ASSERT(allctr->alloc_no == ERTS_ALC_A_LITERAL && allctr->t == 0); - ERTS_SMP_LC_ASSERT(allctr->thread_safe); + ERTS_LC_ASSERT(allctr->thread_safe); erts_alcu_sys_dealloc(allctr, ptr, size, 1); @@ -1191,7 +1155,6 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } -#ifdef ERTS_SMP #ifdef DEBUG static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) @@ -1292,16 +1255,15 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; ERTS_ALC_CPOOL_ASSERT(old_val - == erts_smp_atomic_xchg_relb(&crr->allctr, + == erts_atomic_xchg_relb(&crr->allctr, new_val)); } #else - erts_smp_atomic_set_relb(&crr->allctr, new_val); + erts_atomic_set_relb(&crr->allctr, new_val); #endif } } -#endif /* ERTS_SMP */ #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -1325,12 +1287,10 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) static void *mbc_alloc(Allctr_t *allctr, Uint size); -#ifdef ERTS_SMP typedef struct { ErtsAllctrDDBlock_t ddblock__; /* must be first */ ErtsAlcType_t fix_type; } ErtsAllctrFixDDBlock_t; -#endif #define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) @@ -1341,11 +1301,9 @@ dealloc_fix_block(Allctr_t *allctr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) { -#ifdef ERTS_SMP /* May be redirected... */ ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; -#endif dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); } @@ -1379,12 +1337,10 @@ fix_cpool_check_shrink(Allctr_t *allctr, fix->u.cpool.shrink_list = 0; else { void *p; -#ifdef ERTS_SMP if (busy_pcrr_pp) { clear_busy_pool_carrier(allctr, *busy_pcrr_pp); *busy_pcrr_pp = NULL; } -#endif fix->u.cpool.shrink_list--; p = fix->list; fix->list = *((void **) p); @@ -1477,10 +1433,8 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) int ix, o; int flush = flgs == 0; -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); -#endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { ErtsAlcFixList_t *fix = &allctr->fix[ix]; @@ -1520,10 +1474,8 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) if (all_empty) sched_fix_shrink(allctr, 0); -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); -#endif return res; } @@ -1635,10 +1587,8 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) int ix, o; int flush = flgs == 0; -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); -#endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { ErtsAlcFixList_t *fix = &allctr->fix[ix]; @@ -1680,10 +1630,8 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) if (all_empty) sched_fix_shrink(allctr, 0); -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); -#endif return res; } @@ -1709,7 +1657,6 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) dealloc_carrier(allctr, crr, 1); } -#ifdef ERTS_SMP static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) @@ -1750,7 +1697,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, crr = BLK_TO_SBC(blk); if (sizep) *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; - iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + iallctr = erts_atomic_read_dirty(&crr->allctr); } else { crr = ABLK_TO_MBC(blk); @@ -1758,10 +1705,10 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, if (sizep) *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr)) - iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + iallctr = erts_atomic_read_dirty(&crr->allctr); else { int locked_pref_allctr = 0; - iallctr = erts_smp_atomic_read_ddrb(&crr->allctr); + iallctr = erts_atomic_read_ddrb(&crr->allctr); if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock && pref_allctr->thread_safe) { @@ -1777,7 +1724,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, erts_aint_t act; ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); - act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr, + act = erts_atomic_cmpxchg_ddrb(&crr->allctr, iallctr|ERTS_CRR_ALCTR_FLG_BUSY, iallctr); if (act == iallctr) { @@ -2152,10 +2099,10 @@ handle_delayed_dealloc(Allctr_t *allctr, ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - != (erts_smp_atomic_read_nob(&crr->allctr) + != (erts_atomic_read_nob(&crr->allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); schedule_dealloc_carrier(allctr, crr); } @@ -2201,9 +2148,7 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, erts_alloc_notify_delayed_dealloc(allctr->ix); } -#endif -#ifdef ERTS_SMP static void set_new_allctr_abandon_limit(Allctr_t *allctr); static void @@ -2265,7 +2210,6 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, thr_prgr_p, more_work); } -#endif #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ handle_delayed_dealloc((Allctr), (Locked), 1, \ @@ -2276,24 +2220,18 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ { Block_t *blk = UMEM2BLK(ptr); - ERTS_SMP_LC_ASSERT(!allctr->thread_safe + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); if (IS_SBC_BLK(blk)) { destroy_carrier(allctr, blk, NULL); -#ifdef ERTS_SMP if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; if (!(type & ERTS_ALC_FIX_NO_UNUSE)) fix->u.cpool.used--; fix->u.cpool.allocated--; } -#endif } -#ifndef ERTS_SMP - else - mbc_free(allctr, ptr, NULL); -#else else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) mbc_free(allctr, ptr, NULL); else { @@ -2323,7 +2261,6 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ erts_alloc_notify_delayed_dealloc(used_allctr->ix); } } -#endif } /* Multi block carrier alloc/realloc/free ... */ @@ -2571,9 +2508,7 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) else { (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); -#ifdef ERTS_SMP check_abandon_carrier(allctr, blk, busy_pcrr_pp); -#endif } } @@ -2607,10 +2542,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ -#ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) goto realloc_move; /* Don't want to use carrier in pool */ -#endif get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); @@ -2731,9 +2664,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, HARD_CHECK_BLK_CARRIER(allctr, blk); -#ifdef ERTS_SMP check_abandon_carrier(allctr, nxt_blk, NULL); -#endif return p; } @@ -2845,9 +2776,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ -#ifdef ERTS_SMP realloc_move: -#endif #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); @@ -2949,7 +2878,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #endif /* !MBC_REALLOC_ALWAYS_MOVES */ } -#ifdef ERTS_SMP #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 #define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 @@ -3120,7 +3048,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) == (erts_aint_t) allctr); erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, @@ -3190,7 +3118,7 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); - erts_smp_atomic_set_wb(&crr->allctr, + erts_atomic_set_wb(&crr->allctr, ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); LTTNG3(carrier_pool_put, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, CARRIER_SZ(crr)); } @@ -3322,11 +3250,11 @@ cpool_fetch(Allctr_t *allctr, UWord size) ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); ASSERT(crr->cpool.orig_allctr == allctr); dl = dl->next; - exp = erts_smp_atomic_read_rb(&crr->allctr); + exp = erts_atomic_read_rb(&crr->allctr); if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { /* Try to fetch it... */ - act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + act = erts_atomic_cmpxchg_mb(&crr->allctr, (erts_aint_t) allctr, exp); if (act == exp) { @@ -3368,12 +3296,12 @@ cpool_fetch(Allctr_t *allctr, UWord size) ASSERT(dl != &allctr->cpool.pooled_list); ASSERT(crr->cpool.orig_allctr == allctr); dl = dl->next; - exp = erts_smp_atomic_read_rb(&crr->allctr); + exp = erts_atomic_read_rb(&crr->allctr); if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { /* Try to fetch it... */ - act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + act = erts_atomic_cmpxchg_mb(&crr->allctr, (erts_aint_t) allctr, exp); if (act == exp) { @@ -3449,12 +3377,12 @@ cpool_fetch(Allctr_t *allctr, UWord size) has_passed_sentinel = 1; } crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); - exp = erts_smp_atomic_read_rb(&crr->allctr); + exp = erts_atomic_read_rb(&crr->allctr); if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { erts_aint_t act; /* Try to fetch it... */ - act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + act = erts_atomic_cmpxchg_mb(&crr->allctr, (erts_aint_t) allctr, exp); if (act == exp) { @@ -3477,11 +3405,11 @@ check_dc_list: Block_t* blk; unlink_carrier(&allctr->cpool.dc_list, crr); #ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ERTS_ALC_CPOOL_ASSERT(erts_atomic_xchg_nob(&crr->allctr, ((erts_aint_t) allctr)) == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); #else - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); #endif blk = MBC_TO_FIRST_BLK(allctr, crr); ASSERT(FBLK_TO_MBC(blk) == crr); @@ -3584,7 +3512,7 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) - == (erts_smp_atomic_read_nob(&crr->allctr) + == (erts_atomic_read_nob(&crr->allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) @@ -3735,7 +3663,6 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord * } -#endif /* ERTS_SMP */ #ifdef DEBUG @@ -3836,7 +3763,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); } -#ifdef ERTS_SMP allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC @@ -3852,7 +3778,6 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) return blk; } } -#endif #if HAVE_ERTS_MSEG @@ -3982,9 +3907,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) allctr->main_carrier = crr; } -#ifdef ERTS_SMP cpool_init_carrier_data(allctr, crr); -#endif link_carrier(&allctr->mbc_list, crr); @@ -4204,19 +4127,17 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif -#ifdef ERTS_SMP if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); *busy_pcrr_pp = NULL; - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + ERTS_ALC_CPOOL_ASSERT(erts_atomic_read_nob(&crr->allctr) == (((erts_aint_t) allctr) | ERTS_CRR_ALCTR_FLG_IN_POOL | ERTS_CRR_ALCTR_FLG_BUSY)); - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + erts_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); cpool_delete(allctr, allctr, crr); } else -#endif { unlink_carrier(&allctr->mbc_list, crr); #if HAVE_ERTS_MSEG @@ -4247,11 +4168,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) } #endif -#ifdef ERTS_SMP schedule_dealloc_carrier(allctr, crr); -#else - dealloc_mbc(allctr, crr); -#endif } } @@ -4294,9 +4211,7 @@ static struct { Eterm fix_types; Eterm mbcs; -#ifdef ERTS_SMP Eterm mbcs_pool; -#endif Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -4384,9 +4299,7 @@ init_atoms(Allctr_t *allctr) AM_INIT(fix_types); AM_INIT(mbcs); -#ifdef ERTS_SMP AM_INIT(mbcs_pool); -#endif AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -4636,7 +4549,6 @@ sz_info_carriers(Allctr_t *allctr, return res; } -#ifdef ERTS_SMP static Eterm info_cpool(Allctr_t *allctr, @@ -4690,7 +4602,6 @@ info_cpool(Allctr_t *allctr, return res; } -#endif /* ERTS_SMP */ static Eterm info_carriers(Allctr_t *allctr, @@ -4945,11 +4856,7 @@ info_options(Allctr_t *allctr, return res; } -#ifdef ERTS_SMP acul = allctr->cpool.util_limit; -#else - acul = 0; -#endif if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ @@ -5132,19 +5039,15 @@ erts_alcu_info_options(Allctr_t *allctr, if (hpp || szp) ensure_atoms_initialized(allctr); -#ifdef USE_THREADS if (allctr->thread_safe) { erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); } -#endif res = info_options(allctr, print_to_p, print_to_arg, hpp, szp); -#ifdef USE_THREADS if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); erts_allctr_wrapper_pre_unlock(); } -#endif return res; } @@ -5160,9 +5063,7 @@ erts_alcu_sz_info(Allctr_t *allctr, Uint *szp) { Eterm res, mbcs, sbcs, fix = THE_NON_VALUE; -#ifdef ERTS_SMP Eterm mbcs_pool; -#endif res = THE_NON_VALUE; @@ -5177,12 +5078,10 @@ erts_alcu_sz_info(Allctr_t *allctr, if (hpp || szp) ensure_atoms_initialized(allctr); -#ifdef USE_THREADS if (allctr->thread_safe) { erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); } -#endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); @@ -5198,23 +5097,19 @@ erts_alcu_sz_info(Allctr_t *allctr, fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); -#ifdef ERTS_SMP if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p, print_to_arg, hpp, szp); else mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ -#endif sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); -#ifdef ERTS_SMP if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); -#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_fix_types(allctr, internal, hpp, szp, &res, fix); } @@ -5225,12 +5120,10 @@ erts_alcu_sz_info(Allctr_t *allctr, } -#ifdef USE_THREADS if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); erts_allctr_wrapper_pre_unlock(); } -#endif return res; } @@ -5246,9 +5139,7 @@ erts_alcu_info(Allctr_t *allctr, Uint *szp) { Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE; -#ifdef ERTS_SMP Eterm mbcs_pool; -#endif res = THE_NON_VALUE; @@ -5263,12 +5154,10 @@ erts_alcu_info(Allctr_t *allctr, if (hpp || szp) ensure_atoms_initialized(allctr); -#ifdef USE_THREADS if (allctr->thread_safe) { erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); } -#endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); @@ -5293,13 +5182,11 @@ erts_alcu_info(Allctr_t *allctr, fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); -#ifdef ERTS_SMP if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p, print_to_arg, hpp, szp); else mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ -#endif sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); @@ -5309,10 +5196,8 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); -#ifdef ERTS_SMP if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); -#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); add_fix_types(allctr, internal, hpp, szp, &res, fix); add_2tup(hpp, szp, &res, am.options, sett); @@ -5328,12 +5213,10 @@ erts_alcu_info(Allctr_t *allctr, } -#ifdef USE_THREADS if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); erts_allctr_wrapper_pre_unlock(); } -#endif return res; } @@ -5343,10 +5226,8 @@ void erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) { -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); -#endif size->carriers = allctr->mbcs.curr.norm.mseg.size; size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; @@ -5356,14 +5237,12 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * size->blocks = allctr->mbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; -#ifdef ERTS_SMP if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { UWord csz, bsz; cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); size->blocks += bsz; size->carriers += csz; } -#endif if (fi) { int ix; @@ -5385,10 +5264,8 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * } } -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); -#endif } /* ----------------------------------------------------------------------- */ @@ -5403,7 +5280,7 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) ASSERT(allctr); - ERTS_SMP_LC_ASSERT(!allctr->thread_safe + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); @@ -5436,18 +5313,13 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { void *res; -#ifdef ERTS_SMP ASSERT(!"This is not thread safe"); -#elif defined(USE_THREADS) - ASSERT(erts_equal_tids(erts_main_thread, erts_thr_self())); -#endif res = do_erts_alcu_alloc(type, extra, size); DEBUG_CHECK_ALIGNMENT(res); return res; } -#ifdef USE_THREADS void * erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size) @@ -5463,7 +5335,6 @@ erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size) return res; } -#ifdef ERTS_SMP void * erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size) @@ -5503,21 +5374,17 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); -#ifdef ERTS_SMP ASSERT(pref_allctr->dd.use); ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); -#endif ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); res = do_erts_alcu_alloc(type, pref_allctr, size); -#ifdef ERTS_SMP if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { /* Cleaned up a bit more; try one more time... */ res = do_erts_alcu_alloc(type, pref_allctr, size); } -#endif if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); @@ -5528,9 +5395,7 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) return res; } -#endif -#endif /* ------------------------------------------------------------------------- */ @@ -5543,7 +5408,7 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, ASSERT(allctr); - ERTS_SMP_LC_ASSERT(!allctr->thread_safe + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); @@ -5573,7 +5438,6 @@ void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) do_erts_alcu_free(type, extra, p, NULL); } -#ifdef USE_THREADS void erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) @@ -5584,7 +5448,6 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) erts_mtx_unlock(&allctr->mutex); } -#ifdef ERTS_SMP void erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) @@ -5634,9 +5497,7 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) } } -#endif -#endif /* ------------------------------------------------------------------------- */ @@ -5656,7 +5517,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, ASSERT(allctr); - ERTS_SMP_LC_ASSERT(!allctr->thread_safe + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); @@ -5785,7 +5646,6 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) } -#ifdef USE_THREADS void * erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) @@ -5824,7 +5684,6 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) return res; } -#ifdef ERTS_SMP void * erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, @@ -5905,9 +5764,7 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; Carrier_t *busy_pcrr_p; -#ifdef ERTS_SMP int retried; -#endif if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); @@ -5917,12 +5774,10 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); -#ifdef ERTS_SMP ASSERT(pref_allctr->dd.use); ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); retried = 0; restart: -#endif used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO, p, &old_user_size, &busy_pcrr_p); @@ -5938,13 +5793,11 @@ restart: 0, &busy_pcrr_p); clear_busy_pool_carrier(used_allctr, busy_pcrr_p); -#ifdef ERTS_SMP if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { /* Cleaned up a bit more; try one more time... */ retried = 1; goto restart; } -#endif if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -5999,9 +5852,7 @@ erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra, return realloc_thr_pref(type, extra, p, size, 1); } -#endif -#endif /* ------------------------------------------------------------------------- */ @@ -6022,10 +5873,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) sys_memcpy((void *) &allctr->mseg_opt, (void *) &erts_mseg_default_opt, sizeof(ErtsMsegOpt_t)); -#ifdef ERTS_SMP if (init->tspec || init->tpref) allctr->mseg_opt.sched_spec = 1; -#endif /* ERTS_SMP */ #endif /* HAVE_ERTS_MSEG */ allctr->name_prefix = init->name_prefix; @@ -6083,7 +5932,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? @@ -6107,7 +5955,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; allctr->cpool.util_limit = init->ts ? 0 : init->acul; -#endif allctr->sbc_threshold = init->sbct; #ifndef ARCH_64 @@ -6131,26 +5978,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; #endif -#ifdef USE_THREADS if (init->ts) { allctr->thread_safe = 1; -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_mtx_init_x_opt(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC); -#else - erts_mtx_init_x(&allctr->mutex, - "alcu_allocator", - make_small(allctr->alloc_no)); -#endif /*ERTS_ENABLE_LOCK_COUNT*/ + erts_mtx_init(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), + ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); #ifdef DEBUG allctr->debug.saved_tid = 0; #endif } -#endif if(!allctr->get_free_block || !allctr->link_free_block @@ -6163,14 +6000,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->mbc_header_size < sizeof(Carrier_t)) goto error; -#ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { allctr->dd.use = 1; init_dd_queue(&allctr->dd.q); allctr->dd.ix = init->ix; } -#endif allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + ABLK_HDR_SZ) - ABLK_HDR_SZ); @@ -6224,10 +6059,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) | CFLG_NO_CPOOL | CFLG_MAIN_CARRIER); if (!blk) { -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); -#endif erts_exit(ERTS_ABORT_EXIT, "Failed to create main carrier for %salloc\n", init->name_prefix); @@ -6247,9 +6080,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix[i].type_size = init->fix_type_size[i]; allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; -#ifdef ERTS_SMP ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); -#endif if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { allctr->fix[i].u.cpool.min_list_size = 0; allctr->fix[i].u.cpool.shrink_list = 0; @@ -6269,10 +6100,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) error: -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); -#endif return 0; @@ -6290,10 +6119,8 @@ erts_alcu_stop(Allctr_t *allctr) while (allctr->mbc_list.first) destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL); -#ifdef USE_THREADS if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); -#endif } @@ -6302,14 +6129,12 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { -#ifdef ERTS_SMP int i; for (i = 0; i <= ERTS_ALC_A_MAX; i++) { ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); } -#endif ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); @@ -6324,7 +6149,8 @@ erts_alcu_init(AlcUInit_t *init) carrier_alignment = sizeof(Unit_t); #endif - erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms"); + erts_mtx_init(&init_atoms_mtx, "alcu_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); atoms_initialized = 0; initialized = 1; @@ -6376,7 +6202,6 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; -#ifdef ERTS_SMP case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t); case 0x020: SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1); @@ -6390,14 +6215,6 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) return (UWord) a2; case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1); case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2); -#else - case 0x01f: return (UWord) 0; - case 0x020: return (UWord) 0; - case 0x021: return (UWord) 0; - case 0x022: return (UWord) 0; - case 0x023: return (UWord) 0; - case 0x024: return (UWord) 0; -#endif case 0x025: /* UMEM2BLK_TEST*/ #ifdef DEBUG # ifdef HARD_DEBUG @@ -6455,13 +6272,9 @@ erts_alcu_verify_unused(Allctr_t *allctr) void erts_alcu_verify_unused_ts(Allctr_t *allctr) { -#ifdef USE_THREADS erts_mtx_lock(&allctr->mutex); -#endif erts_alcu_verify_unused(allctr); -#ifdef USE_THREADS erts_mtx_unlock(&allctr->mutex); -#endif } #ifdef DEBUG @@ -6592,3 +6405,45 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) #endif /* ERTS_ALLOC_UTIL_HARD_DEBUG */ +#ifdef ERTS_ENABLE_LOCK_COUNT + +static void lcnt_enable_allocator_lock_count(Allctr_t *allocator, int enable) { + if(!allocator->thread_safe) { + return; + } + + if(enable) { + erts_lcnt_install_new_lock_info(&allocator->mutex.lcnt, + "alcu_allocator", make_small(allocator->alloc_no), + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); + } else { + erts_lcnt_uninstall(&allocator->mutex.lcnt); + } +} + +static void lcnt_update_thread_spec_locks(ErtsAllocatorThrSpec_t *tspec, int enable) { + if(tspec->enabled) { + int i; + + for(i = 0; i < tspec->size; i++) { + lcnt_enable_allocator_lock_count(tspec->allctr[i], enable); + } + } +} + +void erts_lcnt_update_allocator_locks(int enable) { + int i; + + for(i = ERTS_ALC_A_MIN; i < ERTS_ALC_A_MAX; i++) { + ErtsAllocatorInfo_t *ai = &erts_allctrs_info[i]; + + if(ai->enabled && ai->alloc_util) { + if(ai->thr_spec) { + lcnt_update_thread_spec_locks((ErtsAllocatorThrSpec_t*)ai->extra, enable); + } else { + lcnt_enable_allocator_lock_count((Allctr_t*)ai->extra, enable); + } + } + } +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e889980fa4..faeb5ef368 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -24,10 +24,8 @@ #define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" -#ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" -#endif #include "erl_mseg.h" #include "lttng-wrapper.h" @@ -162,12 +160,10 @@ void * erts_alcu_alloc(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free(ErtsAlcType_t, void *, void *); -#ifdef USE_THREADS void * erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_ts(ErtsAlcType_t, void *, void *); -#ifdef ERTS_SMP void * erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint); @@ -176,8 +172,6 @@ void * erts_alcu_alloc_thr_pref(ErtsAlcType_t, void *, Uint); void * erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint); void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); -#endif -#endif Eterm erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *); Eterm erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); @@ -185,9 +179,7 @@ Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); -#ifdef ERTS_SMP void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); -#endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #ifdef ARCH_32 @@ -220,6 +212,10 @@ void* erts_alcu_literal_32_sys_realloc(Allctr_t*, void *ptr, Uint *size_p, Uint void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int superalign); #endif +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_allocator_locks(int enable); +#endif + #endif /* !ERL_ALLOC_UTIL__ */ #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) @@ -234,10 +230,6 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe # endif #endif -#undef MIN -#undef MAX -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) #define FLOOR(X, I) (((X)/(I))*(I)) #define CEILING(X, I) ((((X) - 1)/(I) + 1)*(I)) @@ -304,7 +296,6 @@ void erts_alcu_literal_32_sys_dealloc(Allctr_t*, void *ptr, Uint size, int supe typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; -#ifdef ERTS_SMP typedef struct ErtsDoubleLink_t_ { struct ErtsDoubleLink_t_ *next; @@ -323,21 +314,18 @@ typedef struct { ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ } ErtsAlcCPoolData_t; -#endif typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { UWord chdr; Carrier_t *next; Carrier_t *prev; - erts_smp_atomic_t allctr; -#ifdef ERTS_SMP + erts_atomic_t allctr; ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ -#endif }; #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) typedef struct { Carrier_t *first; @@ -430,7 +418,6 @@ typedef struct { } while (0) #endif -#ifdef ERTS_SMP typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; @@ -473,7 +460,6 @@ typedef struct { } head; } ErtsAllctrDDQueue_t; -#endif typedef struct { size_t type_size; @@ -496,7 +482,6 @@ typedef struct { } ErtsAlcFixList_t; struct Allctr_t_ { -#ifdef ERTS_SMP struct { /* * We want the queue at the beginning of @@ -507,7 +492,6 @@ struct Allctr_t_ { int use; int ix; } dd; -#endif /* Allocator name prefix */ char * name_prefix; @@ -556,7 +540,6 @@ struct Allctr_t_ { /* Carriers */ CarrierList_t mbc_list; CarrierList_t sbc_list; -#ifdef ERTS_SMP struct { /* pooled_list, traitor list and dc_list contain only carriers _created_ by this allocator */ @@ -575,7 +558,6 @@ struct Allctr_t_ { erts_atomic_t no_carriers; } stat; } cpool; -#endif /* Main carrier (if there is one) */ Carrier_t * main_carrier; @@ -618,7 +600,6 @@ struct Allctr_t_ { int fix_shrink_scheduled; ErtsAlcFixList_t *fix; -#ifdef USE_THREADS /* Mutex for this allocator */ erts_mtx_t mutex; int thread_safe; @@ -627,7 +608,6 @@ struct Allctr_t_ { Allctr_t *next; } ts_list; -#endif int atoms_initialized; @@ -650,13 +630,11 @@ struct Allctr_t_ { CarriersStats_t mbcs; #ifdef DEBUG -#ifdef USE_THREADS struct { int saved_tid; erts_tid_t tid; } debug; #endif -#endif }; int erts_alcu_start(Allctr_t *, AllctrInit_t *); @@ -673,7 +651,6 @@ void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); int is_sbc_blk(Block_t*); #endif - #endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__) */ diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c index 861532f241..b6625db0d3 100644 --- a/erts/emulator/beam/erl_arith.c +++ b/erts/emulator/beam/erl_arith.c @@ -114,7 +114,7 @@ BIF_RETTYPE intdiv_2(BIF_ALIST_2) } if (is_both_small(BIF_ARG_1,BIF_ARG_2)){ Sint ires = signed_val(BIF_ARG_1) / signed_val(BIF_ARG_2); - if (MY_IS_SSMALL(ires)) + if (IS_SSMALL(ires)) BIF_RET(make_small(ires)); } BIF_RET(erts_int_div(BIF_P, BIF_ARG_1, BIF_ARG_2)); @@ -276,8 +276,12 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right) goto do_bsl; } else if (is_small(arg1) || is_big(arg1)) { /* - * N bsl PositiveBigNum is too large to represent. + * N bsl PositiveBigNum is too large to represent, + * unless N is 0. */ + if (arg1 == make_small(0)) { + BIF_RET(arg1); + } BIF_ERROR(p, SYSTEM_LIMIT); } /* Fall through if the left argument is not an integer. */ @@ -336,8 +340,7 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) + signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { hp = HAlloc(p, 2); @@ -482,8 +485,7 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) - signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { hp = HAlloc(p, 2); @@ -1177,8 +1179,7 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) + signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { if (ERTS_NEED_GC(p, 2)) { @@ -1345,8 +1346,7 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live) switch ((arg2 & _TAG_IMMED1_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_IMMED1_SMALL >> _TAG_PRIMARY_SIZE): ires = signed_val(arg1) - signed_val(arg2); - ASSERT(MY_IS_SSMALL(ires) == IS_SSMALL(ires)); - if (MY_IS_SSMALL(ires)) { + if (IS_SSMALL(ires)) { return make_small(ires); } else { if (ERTS_NEED_GC(p, 2)) { diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 84254af0c2..3ceb2fd368 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -34,9 +34,6 @@ #define ERTS_ASYNC_PRINT_JOB 0 -#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q -# error "Need async ready queue in non-smp case" -#endif typedef struct _erl_async { DE_Handle* hndl; /* The DE_Handle is needed when port is gone */ @@ -46,16 +43,13 @@ typedef struct _erl_async { ErlDrvPDL pdl; void (*async_invoke)(void*); void (*async_free)(void*); -#if ERTS_USE_ASYNC_READY_Q Uint sched_id; union { ErtsThrQPrepEnQ_t *prep_enq; ErtsThrQFinDeQ_t fin_deq; } q; -#endif } ErtsAsync; -#if ERTS_USE_ASYNC_READY_Q /* * We can do without the enqueue mutex since it isn't needed for @@ -94,7 +88,6 @@ typedef union { char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))]; } ErtsAlgndAsyncReadyQ; -#endif /* ERTS_USE_ASYNC_READY_Q */ typedef struct { ErtsThrQ_t thr_q; @@ -119,12 +112,10 @@ typedef struct { char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))]; } init; ErtsAlgndAsyncQ *queue; -#if ERTS_USE_ASYNC_READY_Q ErtsAlgndAsyncReadyQ *ready_queue; -#endif } ErtsAsyncData; -#if defined(USE_THREADS) && defined(USE_VM_PROBES) +#if defined(USE_VM_PROBES) /* * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace @@ -140,15 +131,6 @@ int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ static ErtsAsyncData *async; -#ifndef USE_THREADS - -void -erts_init_async(void) -{ - -} - -#else static void *async_main(void *); @@ -158,7 +140,6 @@ async_q(int i) return &async->queue[i].aq; } -#if ERTS_USE_ASYNC_READY_Q static ERTS_INLINE ErtsAsyncReadyQ * async_ready_q(Uint sched_id) @@ -166,16 +147,13 @@ async_ready_q(Uint sched_id) return &async->ready_queue[((int)sched_id)-1].arq; } -#endif void erts_init_async(void) { async = NULL; if (erts_async_max_threads > 0) { -#if ERTS_USE_ASYNC_READY_Q ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; -#endif erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; char *ptr, thr_name[16]; size_t tot_size = 0; @@ -183,9 +161,7 @@ erts_init_async(void) tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; -#if ERTS_USE_ASYNC_READY_Q tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; -#endif ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA, tot_size); @@ -194,14 +170,14 @@ erts_init_async(void) ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); async->init.data.no_initialized = 0; - erts_mtx_init(&async->init.data.mtx, "async_init_mtx"); + erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL, + ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); erts_cnd_init(&async->init.data.cnd); erts_atomic_init_nob(&async->init.data.id, 0); async->queue = (ErtsAlgndAsyncQ *) ptr; ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; -#if ERTS_USE_ASYNC_READY_Q qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; @@ -213,14 +189,14 @@ erts_init_async(void) for (i = 1; i <= erts_no_schedulers; i++) { ErtsAsyncReadyQ *arq = async_ready_q(i); #if ERTS_USE_ASYNC_READY_ENQ_MTX - erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx"); + erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); #endif erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq); qinit.arg = (void *) (SWord) i; erts_thr_q_initialize(&arq->thr_q, &qinit); } -#endif /* Create async threads... */ @@ -251,7 +227,6 @@ erts_init_async(void) } } -#if ERTS_USE_ASYNC_READY_Q void * erts_get_async_ready_queue(Uint sched_id) @@ -259,7 +234,6 @@ erts_get_async_ready_queue(Uint sched_id) return (void *) async ? async_ready_q(sched_id) : NULL; } -#endif static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) { @@ -268,10 +242,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif if (is_internal_port(a->port)) { -#if ERTS_USE_ASYNC_READY_Q ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id); a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q); -#endif /* make sure the driver will stay around */ if (a->hndl) erts_ddll_reference_referenced_driver(a->hndl); @@ -307,10 +279,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_t *tse, ErtsThrQPrepEnQ_t **prep_enq) { -#if ERTS_USE_ASYNC_READY_Q int saved_fin_deq = 0; ErtsThrQFinDeQ_t fin_deq; -#endif #ifdef USE_VM_PROBES int len; #endif @@ -319,12 +289,10 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q); if (a) { -#if ERTS_USE_ASYNC_READY_Q *prep_enq = a->q.prep_enq; erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq); if (saved_fin_deq) erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); -#endif #ifdef USE_LTTNG_VM_TRACEPOINTS if (LTTNG_ENABLED(aio_pool_get)) { lttng_decl_portbuf(port_str); @@ -352,7 +320,6 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_reset(tse); -#if ERTS_USE_ASYNC_READY_Q chk_fin_deq: if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) { if (!saved_fin_deq) { @@ -362,13 +329,11 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_thr_q_append_finalize_dequeue_data(&fin_deq, &tmp_fin_deq); } -#endif switch (erts_thr_q_inspect(q, 1)) { case ERTS_THR_Q_DIRTY: break; case ERTS_THR_Q_NEED_THR_PRGR: -#ifdef ERTS_SMP { ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q); erts_thr_progress_wakeup(NULL, prgr); @@ -380,17 +345,14 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_wait(tse); break; } -#endif case ERTS_THR_Q_CLEAN: -#if ERTS_USE_ASYNC_READY_Q if (saved_fin_deq) { if (erts_thr_q_finalize_dequeue(&fin_deq)) goto chk_fin_deq; else saved_fin_deq = 0; } -#endif erts_tse_wait(tse); break; @@ -406,15 +368,10 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, static ERTS_INLINE void call_async_ready(ErtsAsync *a) { -#if ERTS_USE_ASYNC_READY_Q Port *p = erts_id2port_sflgs(a->port, NULL, 0, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); -#else - Port *p = erts_thr_id2port_sflgs(a->port, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); -#endif if (!p) { if (a->async_free) { ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); @@ -430,11 +387,7 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) ERTS_MSACC_POP_STATE(); } } -#if ERTS_USE_ASYNC_READY_Q erts_port_release(p); -#else - erts_thr_port_release(p); -#endif } if (a->pdl) driver_pdl_dec_refc(a->pdl); @@ -444,7 +397,6 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) { -#if ERTS_USE_ASYNC_READY_Q ErtsAsyncReadyQ *arq; #if ERTS_ASYNC_PRINT_JOB @@ -463,12 +415,6 @@ static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) erts_mtx_unlock(&arq->x.data.enq_mtx); #endif -#else /* ERTS_USE_ASYNC_READY_Q */ - - call_async_ready(a); - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - -#endif /* ERTS_USE_ASYNC_READY_Q */ } @@ -484,7 +430,6 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) erts_tse_t *tse = erts_tse_fetch(); ERTS_DECLARE_DUMMY(Uint no); -#ifdef ERTS_SMP ErtsThrPrgrCallbacks callbacks; callbacks.arg = (void *) tse; @@ -493,15 +438,12 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) callbacks.wait = NULL; erts_thr_progress_register_unmanaged_thread(&callbacks); -#endif qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; qinit.arg = (void *) tse; qinit.notify = async_wakeup; -#if ERTS_USE_ASYNC_READY_Q qinit.auto_finalize_dequeue = 0; -#endif erts_thr_q_initialize(&aq->thr_q, &qinit); @@ -543,12 +485,10 @@ static void *async_main(void* arg) return NULL; } -#endif /* USE_THREADS */ void erts_exit_flush_async(void) { -#ifdef USE_THREADS int i; ErtsAsync a; a.port = NIL; @@ -562,11 +502,8 @@ erts_exit_flush_async(void) async_add(&a, async_q(i)); for (i = 0; i < erts_async_max_threads; i++) erts_thr_join(async->queue[i].aq.thr_id, NULL); -#endif } -#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q - int erts_check_async_ready(void *varq) { ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; @@ -607,18 +544,15 @@ int erts_async_ready_clean(void *varq, void *val) case ERTS_THR_Q_DIRTY: return ERTS_ASYNC_READY_DIRTY; case ERTS_THR_Q_NEED_THR_PRGR: -#ifdef ERTS_SMP *((ErtsThrPrgrVal *) val) = erts_thr_q_need_thr_progress(&arq->thr_q); return ERTS_ASYNC_READY_NEED_THR_PRGR; -#endif case ERTS_THR_Q_CLEAN: break; } return ERTS_ASYNC_READY_CLEAN; } -#endif /* ** Generate a fair async key prom an ErlDrvPort @@ -656,28 +590,22 @@ long driver_async(ErlDrvPort ix, unsigned int* key, Port* prt; long id; unsigned int qix; -#if ERTS_USE_ASYNC_READY_Q Uint sched_id; ERTS_MSACC_PUSH_STATE(); sched_id = erts_get_scheduler_id(); if (!sched_id) sched_id = 1; -#else - ERTS_MSACC_PUSH_STATE(); -#endif prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync)); -#if ERTS_USE_ASYNC_READY_Q a->sched_id = sched_id; -#endif a->hndl = (DE_Handle*)prt->drv_ptr->handle; a->port = prt->common.id; a->pdl = NULL; @@ -707,7 +635,6 @@ long driver_async(ErlDrvPort ix, unsigned int* key, (*key % erts_async_max_threads) : 0; *key = qix; } -#ifdef USE_THREADS if (erts_async_max_threads > 0) { if (prt->port_data_lock) { driver_pdl_inc_refc(prt->port_data_lock); @@ -716,7 +643,6 @@ long driver_async(ErlDrvPort ix, unsigned int* key, async_add(a, async_q(qix)); return id; } -#endif ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_invoke)(a->async_data); diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h index 4b470e7679..70ef247e0a 100644 --- a/erts/emulator/beam/erl_async.h +++ b/erts/emulator/beam/erl_async.h @@ -27,39 +27,12 @@ extern int erts_async_max_threads; #define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ extern int erts_async_thread_suggested_stack_size; - -#ifdef ERTS_SMP -/* - * With smp support we can choose to have, or not to - * have an async ready queue. - */ -#define ERTS_USE_ASYNC_READY_Q 1 -#endif - -#ifndef ERTS_SMP -/* In non-smp case we *need* the async ready queue */ -# undef ERTS_USE_ASYNC_READY_Q -# define ERTS_USE_ASYNC_READY_Q 1 -#endif - -#ifndef ERTS_USE_ASYNC_READY_Q -# define ERTS_USE_ASYNC_READY_Q 0 -#endif - -#ifndef USE_THREADS -# undef ERTS_USE_ASYNC_READY_Q -# define ERTS_USE_ASYNC_READY_Q 0 -#endif /* !USE_THREADS */ -#if ERTS_USE_ASYNC_READY_Q int erts_check_async_ready(void *); int erts_async_ready_clean(void *, void *); void *erts_get_async_ready_queue(Uint sched_id); #define ERTS_ASYNC_READY_CLEAN 0 #define ERTS_ASYNC_READY_DIRTY 1 -#ifdef ERTS_SMP #define ERTS_ASYNC_READY_NEED_THR_PRGR 2 -#endif -#endif /* ERTS_USE_ASYNC_READY_Q */ void erts_init_async(void); void erts_exit_flush_async(void); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index 756c7dce05..4cafa499a9 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -171,6 +171,16 @@ static void *my_alloc(MyAllocator *my, Uint size) #define ALPHABET_SIZE 256 +typedef struct _findall_data { + Uint pos; + Uint len; +#ifdef HARDDEBUG + Uint id; +#endif + Eterm epos; + Eterm elen; +} FindallData; + typedef struct _ac_node { #ifdef HARDDEBUG Uint32 id; /* To identify h pointer targets when @@ -208,6 +218,103 @@ typedef struct _bm_data { Sint badshift[ALPHABET_SIZE]; } BMData; +typedef struct _ac_find_all_state { + ACNode *q; + Uint pos; + Uint len; + Uint m; + Uint allocated; + FindallData *out; +} ACFindAllState; + +typedef struct _ac_find_first_state { + ACNode *q; + Uint pos; + Uint len; + ACNode *candidate; + Uint candidate_start; +} ACFindFirstState; + +typedef struct _bm_find_all_state { + Sint pos; + Sint len; + Uint m; + Uint allocated; + FindallData *out; +} BMFindAllState; + +typedef struct _bm_find_first_state { + Sint pos; + Sint len; +} BMFindFirstState; + +typedef enum _bf_return { + BF_RESTART = -3, + BF_NOT_FOUND, + BF_BADARG, + BF_OK +} BFReturn; + +typedef struct _binary_find_all_context { + ErtsHeapFactory factory; + Eterm term; + Sint head; + Sint tail; + Uint end_pos; + Uint size; + FindallData *data; + union { + ACFindAllState ac; + BMFindAllState bm; + } d; +} BinaryFindAllContext; + +typedef struct _binary_find_first_context { + Uint pos; + Uint len; + union { + ACFindFirstState ac; + BMFindFirstState bm; + } d; +} BinaryFindFirstContext; + +typedef struct _binary_find_context BinaryFindContext; + +typedef struct _binary_find_search { + void (*init) (BinaryFindContext *); + BFReturn (*find) (BinaryFindContext *, byte *); + void (*done) (BinaryFindContext *); +} BinaryFindSearch; + +typedef Eterm (*BinaryFindResult)(Process *, Eterm, BinaryFindContext **); + +typedef enum _binary_find_state { + BFSearch, + BFResult, + BFDone +} BinaryFindState; + +struct _binary_find_context { + Eterm pat_type; + Eterm pat_term; + Binary *pat_bin; + Uint flags; + Uint hsstart; + Uint hsend; + int loop_factor; + int exported; + Uint reds; + BinaryFindState state; + Eterm trap_term; + BinaryFindSearch *search; + BinaryFindResult not_found; + BinaryFindResult found; + union { + BinaryFindAllContext fa; + BinaryFindFirstContext ff; + } u; +}; + #ifdef HARDDEBUG static void dump_bm_data(BMData *bm); static void dump_ac_trie(ACTrie *act); @@ -229,13 +336,6 @@ static void dump_ac_node(ACNode *node, int indent, int ch); MYALIGN(sizeof(ACTrie))) /* Structure */ -#ifndef MAX -#define MAX(A,B) (((A) > (B)) ? (A) : (B)) -#endif - -#ifndef MIN -#define MIN(A,B) (((A) > (B)) ? (B) : (A)) -#endif /* * Callback for the magic binary */ @@ -421,32 +521,25 @@ static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) * Basic AC finds the first end before the first start... * */ -typedef struct { - ACNode *q; - Uint pos; - Uint len; - ACNode *candidate; - Uint candidate_start; -} ACFindFirstState; - - -static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len) +static void ac_init_find_first_match(BinaryFindContext *ctx) { + ACFindFirstState *state = &(ctx->u.ff.d.ac); + ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); state->q = act->root; - state->pos = startpos; - state->len = len; + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->candidate = NULL; state->candidate_start = 0; } -#define AC_OK 0 -#define AC_NOT_FOUND -1 -#define AC_RESTART -2 #define AC_LOOP_FACTOR 10 -static int ac_find_first_match(ACFindFirstState *state, byte *haystack, - Uint *mpos, Uint *mlen, Uint *reductions) +static BFReturn ac_find_first_match(BinaryFindContext *ctx, byte *haystack) { + ACFindFirstState *state = &(ctx->u.ff.d.ac); + Uint *mpos = &(ctx->u.ff.pos); + Uint *mlen = &(ctx->u.ff.len); + Uint *reductions = &(ctx->reds); ACNode *q = state->q; Uint i = state->pos; ACNode *candidate = state->candidate, *r; @@ -462,7 +555,7 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack, state->len = len; state->candidate = candidate; state->candidate_start = candidate_start; - return AC_RESTART; + return BF_RESTART; } while (q->g[haystack[i]] == NULL && q->h != q) { @@ -492,68 +585,33 @@ static int ac_find_first_match(ACFindFirstState *state, byte *haystack, } *reductions = reds; if (!candidate) { - return AC_NOT_FOUND; + return BF_NOT_FOUND; } #ifdef HARDDEBUG dump_ac_node(candidate,0,'?'); #endif *mpos = candidate_start; *mlen = candidate->d; - return AC_OK; + return BF_OK; } -typedef struct _findall_data { - Uint pos; - Uint len; -#ifdef HARDDEBUG - Uint id; -#endif - Eterm epos; - Eterm elen; -} FindallData; - -typedef struct { - ACNode *q; - Uint pos; - Uint len; - Uint m; - Uint allocated; - FindallData *out; -} ACFindAllState; - -static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len) +static void ac_init_find_all(BinaryFindContext *ctx) { + ACFindAllState *state = &(ctx->u.fa.d.ac); + ACTrie *act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); state->q = act->root; - state->pos = startpos; - state->len = len; + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->m = 0; state->allocated = 0; state->out = NULL; } -static void ac_restore_find_all(ACFindAllState *state, - const ACFindAllState *src) -{ - memcpy(state, src, sizeof(ACFindAllState)); - if (state->allocated > 0) { - state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); - memcpy(state->out, src+1, sizeof(FindallData)*state->m); - } else { - state->out = NULL; - } -} - -static void ac_serialize_find_all(const ACFindAllState *state, - ACFindAllState *dst) -{ - memcpy(dst, state, sizeof(ACFindAllState)); - memcpy(dst+1, state->out, sizeof(FindallData)*state->m); -} - -static void ac_clean_find_all(ACFindAllState *state) +static void ac_clean_find_all(BinaryFindContext *ctx) { + ACFindAllState *state = &(ctx->u.fa.d.ac); if (state->out != NULL) { - erts_free(ERTS_ALC_T_TMP, state->out); + erts_free(ERTS_ALC_T_BINARY_FIND, state->out); } #ifdef HARDDEBUG state->out = NULL; @@ -565,9 +623,10 @@ static void ac_clean_find_all(ACFindAllState *state) * Differs to the find_first function in that it stores all matches and the values * arte returned only in the state. */ -static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, - Uint *reductions) +static BFReturn ac_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack) { + ACFindAllState *state = &(ctx->u.fa.d.ac); + Uint *reductions = &(ctx->reds); ACNode *q = state->q; Uint i = state->pos; Uint rstart; @@ -578,7 +637,6 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, FindallData *out = state->out; register Uint reds = *reductions; - while (i < len) { if (--reds == 0) { state->q = q; @@ -587,7 +645,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, state->m = m; state->allocated = allocated; state->out = out; - return AC_RESTART; + return BF_RESTART; } while (q->g[haystack[i]] == NULL && q->h != q) { q = q->h; @@ -625,11 +683,11 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, if (m >= allocated) { if (!allocated) { allocated = 10; - out = erts_alloc(ERTS_ALC_T_TMP, + out = erts_alloc(ERTS_ALC_T_BINARY_FIND, sizeof(FindallData) * allocated); } else { allocated *= 2; - out = erts_realloc(ERTS_ALC_T_TMP, out, + out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out, sizeof(FindallData) * allocated); } @@ -656,7 +714,7 @@ static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, *reductions = reds; state->m = m; state->out = out; - return (m == 0) ? AC_NOT_FOUND : AC_OK; + return (m == 0) ? BF_NOT_FOUND : BF_OK; } /* @@ -743,27 +801,22 @@ static void compute_goodshifts(BMData *bmd) erts_free(ERTS_ALC_T_TMP, suffixes); } -typedef struct { - Sint pos; - Sint len; -} BMFindFirstState; - -#define BM_OK 0 /* used only for find_all */ -#define BM_NOT_FOUND -1 -#define BM_RESTART -2 #define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ -static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos, - Uint len) +static void bm_init_find_first_match(BinaryFindContext *ctx) { - state->pos = startpos; - state->len = (Sint) len; + BMFindFirstState *state = &(ctx->u.ff.d.bm); + state->pos = ctx->hsstart; + state->len = ctx->hsend; } - -static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, - byte *haystack, Uint *reductions) +static BFReturn bm_find_first_match(BinaryFindContext *ctx, byte *haystack) { + BMFindFirstState *state = &(ctx->u.ff.d.bm); + BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + Uint *mpos = &(ctx->u.ff.pos); + Uint *mlen = &(ctx->u.ff.len); + Uint *reductions = &(ctx->reds); Sint blen = bmd->len; Sint len = state->len; Sint *gs = bmd->goodshift; @@ -776,61 +829,37 @@ static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, while (j <= len - blen) { if (--reds == 0) { state->pos = j; - return BM_RESTART; + return BF_RESTART; } for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) ; if (i < 0) { /* found */ *reductions = reds; - return j; + *mpos = (Uint) j; + *mlen = (Uint) blen; + return BF_OK; } j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); } *reductions = reds; - return BM_NOT_FOUND; + return BF_NOT_FOUND; } -typedef struct { - Sint pos; - Sint len; - Uint m; - Uint allocated; - FindallData *out; -} BMFindAllState; - -static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) +static void bm_init_find_all(BinaryFindContext *ctx) { - state->pos = startpos; - state->len = (Sint) len; + BMFindAllState *state = &(ctx->u.fa.d.bm); + state->pos = ctx->hsstart; + state->len = ctx->hsend; state->m = 0; state->allocated = 0; state->out = NULL; } -static void bm_restore_find_all(BMFindAllState *state, - const BMFindAllState *src) -{ - memcpy(state, src, sizeof(BMFindAllState)); - if (state->allocated > 0) { - state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * - (state->allocated)); - memcpy(state->out, src+1, sizeof(FindallData)*state->m); - } else { - state->out = NULL; - } -} - -static void bm_serialize_find_all(const BMFindAllState *state, - BMFindAllState *dst) -{ - memcpy(dst, state, sizeof(BMFindAllState)); - memcpy(dst+1, state->out, sizeof(FindallData)*state->m); -} - -static void bm_clean_find_all(BMFindAllState *state) +static void bm_clean_find_all(BinaryFindContext *ctx) { + BMFindAllState *state = &(ctx->u.fa.d.bm); if (state->out != NULL) { - erts_free(ERTS_ALC_T_TMP, state->out); + erts_free(ERTS_ALC_T_BINARY_FIND, state->out); } #ifdef HARDDEBUG state->out = NULL; @@ -842,10 +871,11 @@ static void bm_clean_find_all(BMFindAllState *state) * Differs to the find_first function in that it stores all matches and the * values are returned only in the state. */ -static Sint bm_find_all_non_overlapping(BMFindAllState *state, - BMData *bmd, byte *haystack, - Uint *reductions) +static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haystack) { + BMFindAllState *state = &(ctx->u.fa.d.bm); + BMData *bmd = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + Uint *reductions = &(ctx->reds); Sint blen = bmd->len; Sint len = state->len; Sint *gs = bmd->goodshift; @@ -864,7 +894,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, state->m = m; state->allocated = allocated; state->out = out; - return BM_RESTART; + return BF_RESTART; } for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) ; @@ -872,10 +902,11 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, if (m >= allocated) { if (!allocated) { allocated = 10; - out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated); + out = erts_alloc(ERTS_ALC_T_BINARY_FIND, + sizeof(FindallData) * allocated); } else { allocated *= 2; - out = erts_realloc(ERTS_ALC_T_TMP, out, + out = erts_realloc(ERTS_ALC_T_BINARY_FIND, out, sizeof(FindallData) * allocated); } } @@ -890,7 +921,7 @@ static Sint bm_find_all_non_overlapping(BMFindAllState *state, state->m = m; state->out = out; *reductions = reds; - return (m == 0) ? BM_NOT_FOUND : BM_OK; + return (m == 0) ? BF_NOT_FOUND : BF_OK; } /* @@ -1016,51 +1047,160 @@ BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) BIF_RET(ret); } -#define DO_BIN_MATCH_OK 0 -#define DO_BIN_MATCH_BADARG -1 -#define DO_BIN_MATCH_RESTART -2 +#define BF_FLAG_GLOBAL 0x01 +#define BF_FLAG_SPLIT_TRIM 0x02 +#define BF_FLAG_SPLIT_TRIM_ALL 0x04 -#define BINARY_FIND_ALL 0x01 -#define BINARY_SPLIT_TRIM 0x02 -#define BINARY_SPLIT_TRIM_ALL 0x04 +static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found, + BinaryFindResult single, BinaryFindResult global, + Binary *pat_bin); +static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src); +static int bf_context_destructor(Binary *ctx_bin); +#ifdef HARDDEBUG +static void bf_context_dump(BinaryFindContext *ctx); +#endif -typedef struct BinaryFindState { - Eterm type; - Uint flags; - Uint hsstart; - Uint hsend; - Eterm (*not_found_result) (Process *, Eterm, struct BinaryFindState *); - Eterm (*single_result) (Process *, Eterm, struct BinaryFindState *, Sint, Sint); - Eterm (*global_result) (Process *, Eterm, struct BinaryFindState *, FindallData *, Uint); -} BinaryFindState; +static BinaryFindSearch bf_search_ac_global = { + ac_init_find_all, + ac_find_all_non_overlapping, + ac_clean_find_all +}; + +static BinaryFindSearch bf_search_ac_single = { + ac_init_find_first_match, + ac_find_first_match, + NULL +}; + +static BinaryFindSearch bf_search_bm_global = { + bm_init_find_all, + bm_find_all_non_overlapping, + bm_clean_find_all +}; + +static BinaryFindSearch bf_search_bm_single = { + bm_init_find_first_match, + bm_find_first_match, + NULL +}; + +static void bf_context_init(BinaryFindContext *ctx, BinaryFindResult not_found, + BinaryFindResult single, BinaryFindResult global, + Binary *pat_bin) +{ + ctx->exported = 0; + ctx->state = BFSearch; + ctx->not_found = not_found; + if (ctx->flags & BF_FLAG_GLOBAL) { + ctx->found = global; + if (ctx->pat_type == am_bm) { + ctx->search = &bf_search_bm_global; + ctx->loop_factor = BM_LOOP_FACTOR; + } else if (ctx->pat_type == am_ac) { + ctx->search = &bf_search_ac_global; + ctx->loop_factor = AC_LOOP_FACTOR; + } + } else { + ctx->found = single; + if (ctx->pat_type == am_bm) { + ctx->search = &bf_search_bm_single; + ctx->loop_factor = BM_LOOP_FACTOR; + } else if (ctx->pat_type == am_ac) { + ctx->search = &bf_search_ac_single; + ctx->loop_factor = AC_LOOP_FACTOR; + } + } + ctx->trap_term = THE_NON_VALUE; + ctx->pat_bin = pat_bin; + ctx->search->init(ctx); +} -typedef struct BinaryFindState_bignum { - Eterm bignum_hdr; - BinaryFindState bfs; - union { - BMFindFirstState bmffs; - BMFindAllState bmfas; - ACFindFirstState acffs; - ACFindAllState acfas; - } data; -} BinaryFindState_bignum; - -#define SIZEOF_BINARY_FIND_STATE(S) \ - (sizeof(BinaryFindState)+sizeof(S)) - -#define SIZEOF_BINARY_FIND_ALL_STATE(S) \ - (sizeof(BinaryFindState)+sizeof(S)+(sizeof(FindallData)*(S).m)) - -static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); -static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len); -static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz); -static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs); -static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len); -static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz); +static BinaryFindContext *bf_context_export(Process *p, BinaryFindContext *src) +{ + Binary *ctx_bin; + BinaryFindContext *ctx; + Eterm *hp; + + ASSERT(src->exported == 0); + ctx_bin = erts_create_magic_binary(sizeof(BinaryFindContext), + bf_context_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + sys_memcpy(ctx, src, sizeof(BinaryFindContext)); + if (ctx->pat_bin != NULL && ctx->pat_term == THE_NON_VALUE) { + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE * 2); + ctx->pat_term = erts_mk_magic_ref(&hp, &MSO(p), ctx->pat_bin); + } else { + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + } + ctx->trap_term = erts_mk_magic_ref(&hp, &MSO(p), ctx_bin); + ctx->exported = 1; + return ctx; +} + +static int bf_context_destructor(Binary *ctx_bin) +{ + BinaryFindContext *ctx; + + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + if (ctx->state != BFDone) { + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + } + return 1; +} + +#ifdef HARDDEBUG +static void bf_context_dump(BinaryFindContext *ctx) +{ + if (ctx->pat_type == am_bm) { + BMData *bm; + bm = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + dump_bm_data(bm); + } else { + ACTrie *act; + act = ERTS_MAGIC_BIN_DATA(ctx->pat_bin); + dump_ac_trie(act); + } +} +#endif + +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp); +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp); + +static BFReturn maybe_binary_match_compile(BinaryFindContext *ctx, Eterm arg, Binary **pat_bin) +{ + Eterm *tp; + ctx->pat_term = THE_NON_VALUE; + if (is_tuple(arg)) { + tp = tuple_val(arg); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + return BF_BADARG; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !is_internal_magic_ref(tp[2])) { + return BF_BADARG; + } + *pat_bin = erts_magic_ref2bin(tp[2]); + if ((tp[1] == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_bm) || + (tp[1] == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(*pat_bin) != cleanup_my_data_ac)) { + *pat_bin = NULL; + return BF_BADARG; + } + ctx->pat_type = tp[1]; + ctx->pat_term = tp[2]; + } else if (do_binary_match_compile(arg, &(ctx->pat_type), pat_bin) != 0) { + return BF_BADARG; + } + return BF_OK; +} static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) { @@ -1141,17 +1281,17 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin Uint orig_size; if (is_atom(t)) { if (t == am_global) { - *optp |= BINARY_FIND_ALL; + *optp |= BF_FLAG_GLOBAL; l = CDR(list_val(l)); continue; } if (t == am_trim) { - *optp |= BINARY_SPLIT_TRIM; + *optp |= BF_FLAG_SPLIT_TRIM; l = CDR(list_val(l)); continue; } if (t == am_trim_all) { - *optp |= BINARY_SPLIT_TRIM_ALL; + *optp |= BF_FLAG_SPLIT_TRIM_ALL; l = CDR(list_val(l)); continue; } @@ -1204,266 +1344,160 @@ static int parse_split_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp, Uin } } -static int do_binary_find(Process *p, Eterm subject, BinaryFindState *bfs, Binary *bin, - Eterm state_term, Eterm *res_term) +static BFReturn do_binary_find(Process *p, Eterm subject, BinaryFindContext **ctxp, + Binary *pat_bin, Binary *ctx_bin, Eterm *res_term) { - byte *bytes; - Uint bitoffs, bitsize; - byte *temp_alloc = NULL; - BinaryFindState_bignum *state_ptr = NULL; + BinaryFindContext *ctx; + int is_first_call; + Uint initial_reds; + BFReturn runres; - ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); - if (bitsize != 0) { - goto badarg; - } - if (bitoffs != 0) { - bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); - } - if (state_term != NIL) { - state_ptr = (BinaryFindState_bignum *)(big_val(state_term)); - bfs = &(state_ptr->bfs); + if (ctx_bin == NULL) { + is_first_call = 1; + ctx = *ctxp; + } else { + is_first_call = 0; + ctx = ERTS_MAGIC_BIN_DATA(ctx_bin); + ctx->pat_bin = pat_bin; + *ctxp = ctx; } - if (bfs->flags & BINARY_FIND_ALL) { - if (bfs->type == am_bm) { - BMData *bm; - Sint pos; - BMFindAllState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; + initial_reds = ctx->reds = get_reds(p, ctx->loop_factor); - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_all(&state, bfs->hsstart, bfs->hsend); - } else { - bm_restore_find_all(&state, &(state_ptr->data.bmfas)); - } + switch (ctx->state) { + case BFSearch: { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; - pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap bm!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - bm_serialize_find_all(&state, &state_ptr->data.bmfas); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); - } - erts_free_aligned_binary_bytes(temp_alloc); - bm_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } else if (bfs->type == am_ac) { - ACTrie *act; - int acr; - ACFindAllState state; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } #ifdef HARDDEBUG - dump_ac_trie(act); + bf_context_dump(ctx); #endif - if (state_term == NIL) { - ac_init_find_all(&state, act, bfs->hsstart, bfs->hsend); - } else { - ac_restore_find_all(&state, &(state_ptr->data.acfas)); - } - acr = ac_find_all_non_overlapping(&state, bytes, &reds); - if (acr == AC_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_BINARY_FIND_ALL_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_ALL_STATE(state) % sizeof(Eterm)); + runres = ctx->search->find(ctx, bytes); + if (runres == BF_NOT_FOUND) { + *res_term = ctx->not_found(p, subject, &ctx); + *ctxp = ctx; + } else if (runres == BF_RESTART) { #ifdef HARDDEBUG + if (ctx->pat_type == am_ac) { erts_printf("Trap ac!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - ac_serialize_find_all(&state, &state_ptr->data.acfas); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->global_result(p, subject, bfs, state.out, state.m); - } - erts_free_aligned_binary_bytes(temp_alloc); - ac_clean_find_all(&state); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } - } else { - if (bfs->type == am_bm) { - BMData *bm; - Sint pos; - BMFindFirstState state; - Uint reds = get_reds(p, BM_LOOP_FACTOR); - Uint save_reds = reds; - - bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_bm_data(bm); -#endif - if (state_term == NIL) { - bm_init_find_first_match(&state, bfs->hsstart, bfs->hsend); } else { - memcpy(&state, &state_ptr->data.bmffs, sizeof(BMFindFirstState)); - } - -#ifdef HARDDEBUG - erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, - state.len); -#endif - pos = bm_find_first_match(&state, bm, bytes, &reds); - if (pos == BM_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (pos == BM_RESTART) { - int x = - (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG erts_printf("Trap bm!\n"); + } #endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - memcpy(&state_ptr->data.acffs, &state, sizeof(BMFindFirstState)); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->single_result(p, subject, bfs, pos, bm->len); + if (is_first_call) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + erts_set_gc_state(p, 0); } erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); - return DO_BIN_MATCH_OK; - } else if (bfs->type == am_ac) { - ACTrie *act; - Uint pos, rlen; - int acr; - ACFindFirstState state; - Uint reds = get_reds(p, AC_LOOP_FACTOR); - Uint save_reds = reds; - - act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); -#ifdef HARDDEBUG - dump_ac_trie(act); -#endif - if (state_term == NIL) { - ac_init_find_first_match(&state, act, bfs->hsstart, bfs->hsend); - } else { - memcpy(&state, &state_ptr->data.acffs, sizeof(ACFindFirstState)); + *res_term = THE_NON_VALUE; + BUMP_ALL_REDS(p); + return BF_RESTART; + } else { + *res_term = ctx->found(p, subject, &ctx); + *ctxp = ctx; + } + erts_free_aligned_binary_bytes(temp_alloc); + if (*res_term == THE_NON_VALUE) { + if (is_first_call) { + erts_set_gc_state(p, 0); } - acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); - if (acr == AC_NOT_FOUND) { - *res_term = bfs->not_found_result(p, subject, bfs); - } else if (acr == AC_RESTART) { - int x = - (SIZEOF_BINARY_FIND_STATE(state) / sizeof(Eterm)) + - !!(SIZEOF_BINARY_FIND_STATE(state) % sizeof(Eterm)); -#ifdef HARDDEBUG - erts_printf("Trap ac!\n"); -#endif - state_ptr = (BinaryFindState_bignum*) HAlloc(p, x+1); - state_ptr->bignum_hdr = make_pos_bignum_header(x); - memcpy(&state_ptr->bfs, bfs, sizeof(BinaryFindState)); - memcpy(&state_ptr->data.acffs, &state, sizeof(ACFindFirstState)); - *res_term = make_big(&state_ptr->bignum_hdr); - erts_free_aligned_binary_bytes(temp_alloc); - return DO_BIN_MATCH_RESTART; - } else { - *res_term = bfs->single_result(p, subject, bfs, pos, rlen); + BUMP_ALL_REDS(p); + return BF_RESTART; + } + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + if (!is_first_call) { + erts_set_gc_state(p, 1); + } + BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor); + return BF_OK; + } + case BFResult: { + *res_term = ctx->found(p, subject, &ctx); + *ctxp = ctx; + if (*res_term == THE_NON_VALUE) { + if (is_first_call) { + erts_set_gc_state(p, 0); } - erts_free_aligned_binary_bytes(temp_alloc); - BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); - return DO_BIN_MATCH_OK; + BUMP_ALL_REDS(p); + return BF_RESTART; } + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + if (!is_first_call) { + erts_set_gc_state(p, 1); + } + BUMP_REDS(p, (initial_reds - ctx->reds) / ctx->loop_factor); + return BF_OK; } - badarg: - return DO_BIN_MATCH_BADARG; + default: + ASSERT(!"Unknown state in do_binary_find"); + } + +badarg: + if (!is_first_call) { + if (ctx->search->done != NULL) { + ctx->search->done(ctx); + } + ctx->state = BFDone; + erts_set_gc_state(p, 1); + } + return BF_BADARG; } static BIF_RETTYPE binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, Uint flags) { - BinaryFindState bfs; - Eterm *tp; - Binary *bin; - Eterm bin_term = NIL; + BinaryFindContext c_buff; + BinaryFindContext *ctx = &c_buff; + Binary *pat_bin; int runres; Eterm result; - if (is_not_binary(arg1)) { + if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) { goto badarg; } - bfs.flags = flags; - if (parse_match_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend))) { + ctx->flags = flags; + if (parse_match_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend))) { goto badarg; } - if (bfs.hsend == 0) { - BIF_RET(do_match_not_found_result(p, arg1, &bfs)); + if (ctx->hsend == 0) { + result = do_match_not_found_result(p, arg1, &ctx); + BIF_RET(result); } - if (is_tuple(arg2)) { - tp = tuple_val(arg2); - if (arityval(*tp) != 2 || is_not_atom(tp[1])) { - goto badarg; - } - if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !is_internal_magic_ref(tp[2])) { - goto badarg; - } - bfs.type = tp[1]; - bin = erts_magic_ref2bin(tp[2]); - if (bfs.type == am_bm && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { - goto badarg; - } - if (bfs.type == am_ac && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { - goto badarg; - } - bin_term = tp[2]; - } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { + if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) { goto badarg; } - bfs.not_found_result = &do_match_not_found_result; - bfs.single_result = &do_match_single_result; - bfs.global_result = &do_match_global_result; - runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); - if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); - } else if (bin_term == NIL) { - erts_bin_free(bin); + bf_context_init(ctx, do_match_not_found_result, do_match_single_result, + do_match_global_result, pat_bin); + runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result); + if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) { + erts_bin_free(pat_bin); } switch (runres) { - case DO_BIN_MATCH_OK: + case BF_OK: BIF_RET(result); - case DO_BIN_MATCH_RESTART: - BUMP_ALL_REDS(p); - BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); + case BF_RESTART: + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term); default: goto badarg; } - badarg: - BIF_ERROR(p,BADARG); +badarg: + BIF_ERROR(p, BADARG); } BIF_RETTYPE binary_match_2(BIF_ALIST_2) @@ -1478,76 +1512,52 @@ BIF_RETTYPE binary_match_3(BIF_ALIST_3) BIF_RETTYPE binary_matches_2(BIF_ALIST_2) { - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BINARY_FIND_ALL); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, THE_NON_VALUE, BF_FLAG_GLOBAL); } BIF_RETTYPE binary_matches_3(BIF_ALIST_3) { - return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BINARY_FIND_ALL); + return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, BF_FLAG_GLOBAL); } static BIF_RETTYPE binary_split(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) { - BinaryFindState bfs; - Eterm *tp; - Binary *bin; - Eterm bin_term = NIL; + BinaryFindContext c_buff; + BinaryFindContext *ctx = &c_buff; + Binary *pat_bin; int runres; Eterm result; - if (is_not_binary(arg1)) { + if (is_not_binary(arg1) || binary_bitsize(arg1) != 0) { goto badarg; } - if (parse_split_opts_list(arg3, arg1, &(bfs.hsstart), &(bfs.hsend), &(bfs.flags))) { + if (parse_split_opts_list(arg3, arg1, &(ctx->hsstart), &(ctx->hsend), &(ctx->flags))) { goto badarg; } - if (bfs.hsend == 0) { - result = do_split_not_found_result(p, arg1, &bfs); + if (ctx->hsend == 0) { + result = do_split_not_found_result(p, arg1, &ctx); BIF_RET(result); } - if (is_tuple(arg2)) { - tp = tuple_val(arg2); - if (arityval(*tp) != 2 || is_not_atom(tp[1])) { - goto badarg; - } - if (((tp[1] != am_bm) && (tp[1] != am_ac)) || - !is_internal_magic_ref(tp[2])) { - goto badarg; - } - bfs.type = tp[1]; - bin = erts_magic_ref2bin(tp[2]); - if (bfs.type == am_bm && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { - goto badarg; - } - if (bfs.type == am_ac && - ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { - goto badarg; - } - bin_term = tp[2]; - } else if (do_binary_match_compile(arg2, &(bfs.type), &bin)) { + if (maybe_binary_match_compile(ctx, arg2, &pat_bin) != BF_OK) { goto badarg; } - bfs.not_found_result = &do_split_not_found_result; - bfs.single_result = &do_split_single_result; - bfs.global_result = &do_split_global_result; - runres = do_binary_find(p, arg1, &bfs, bin, NIL, &result); - if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { - Eterm *hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); - bin_term = erts_mk_magic_ref(&hp, &MSO(p), bin); - } else if (bin_term == NIL) { - erts_bin_free(bin); - } - switch(runres) { - case DO_BIN_MATCH_OK: + bf_context_init(ctx, do_split_not_found_result, do_split_single_result, + do_split_global_result, pat_bin); + runres = do_binary_find(p, arg1, &ctx, pat_bin, NULL, &result); + if (runres == BF_OK && ctx->pat_term == THE_NON_VALUE) { + erts_bin_free(pat_bin); + } + switch (runres) { + case BF_OK: BIF_RET(result); - case DO_BIN_MATCH_RESTART: - BIF_TRAP3(&binary_find_trap_export, p, arg1, result, bin_term); + case BF_RESTART: + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, p, arg1, ctx->trap_term, ctx->pat_term); default: goto badarg; } - badarg: +badarg: BIF_ERROR(p, BADARG); } @@ -1561,72 +1571,117 @@ BIF_RETTYPE binary_split_3(BIF_ALIST_3) return binary_split(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } -static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +static Eterm do_match_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - if (bfs->flags & BINARY_FIND_ALL) { + if ((*ctxp)->flags & BF_FLAG_GLOBAL) { return NIL; } else { return am_nomatch; } } -static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len) +static Eterm do_match_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); + BinaryFindFirstContext *ff = &(ctx->u.ff); Eterm erlen; Eterm *hp; Eterm ret; - erlen = erts_make_integer((Uint)(len), p); - ret = erts_make_integer(pos, p); + erlen = erts_make_integer((Uint)(ff->len), p); + ret = erts_make_integer(ff->pos, p); hp = HAlloc(p, 3); ret = TUPLE2(hp, ret, erlen); return ret; } -static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz) +static Eterm do_match_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - Sint i; + BinaryFindContext *ctx = (*ctxp); + BinaryFindAllContext *fa = &(ctx->u.fa); + FindallData *fad; Eterm tpl; - Eterm *hp; - Eterm ret; + Sint i; + register Uint reds = ctx->reds; - for (i = 0; i < fad_sz; ++i) { - fad[i].epos = erts_make_integer(fad[i].pos, p); - fad[i].elen = erts_make_integer(fad[i].len, p); + if (ctx->state == BFSearch) { + if (ctx->pat_type == am_ac) { + fa->data = fa->d.ac.out; + fa->size = fa->d.ac.m; + } else { + fa->data = fa->d.bm.out; + fa->size = fa->d.bm.m; + } + fa->tail = fa->size - 1; + fa->head = 0; + fa->end_pos = 0; + fa->term = NIL; + if (ctx->exported == 0 && ((fa->size * 2) >= reds)) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + fa = &(ctx->u.fa); + } + erts_factory_proc_prealloc_init(&(fa->factory), p, fa->size * (3 + 2)); + ctx->state = BFResult; + } + + fad = fa->data; + + if (fa->end_pos == 0) { + for (i = fa->head; i < fa->size; ++i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + return THE_NON_VALUE; + } + fad[i].epos = erts_make_integer(fad[i].pos, p); + fad[i].elen = erts_make_integer(fad[i].len, p); + } + fa->end_pos = 1; + fa->head = fa->tail; } - hp = HAlloc(p, fad_sz * (3 + 2)); - ret = NIL; - for (i = fad_sz - 1; i >= 0; --i) { - tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); - hp += 3; - ret = CONS(hp, tpl, ret); - hp += 2; + + for (i = fa->head; i >= 0; --i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + return THE_NON_VALUE; + } + tpl = TUPLE2(fa->factory.hp, fad[i].epos, fad[i].elen); + fa->factory.hp += 3; + fa->term = CONS(fa->factory.hp, tpl, fa->term); + fa->factory.hp += 2; } + ctx->reds = reds; + erts_factory_close(&(fa->factory)); - return ret; + return fa->term; } -static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindState *bfs) +static Eterm do_split_not_found_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); Eterm *hp; Eterm ret; - if (bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL) + if (ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL) && binary_size(subject) == 0) { - return NIL; + return NIL; } hp = HAlloc(p, 2); ret = CONS(hp, subject, NIL); - return ret; } -static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState *bfs, - Sint pos, Sint len) +static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { + BinaryFindContext *ctx = (*ctxp); + BinaryFindFirstContext *ff = &(ctx->u.ff); + Sint pos; + Sint len; size_t orig_size; Eterm orig; Uint offset; @@ -1637,9 +1692,12 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * Eterm *hp; Eterm ret; + pos = ff->pos; + len = ff->len; + orig_size = binary_size(subject); - if ((bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL)) && + if ((ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL)) && (orig_size - pos - len) == 0) { if (pos == 0) { ret = NIL; @@ -1660,7 +1718,7 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * hp += 2; } } else { - if ((bfs->flags & BINARY_SPLIT_TRIM_ALL) && (pos == 0)) { + if ((ctx->flags & BF_FLAG_SPLIT_TRIM_ALL) && (pos == 0)) { hp = HAlloc(p, 1 * (ERL_SUB_BIN_SIZE + 2)); ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); sb1 = NULL; @@ -1698,39 +1756,60 @@ static Eterm do_split_single_result(Process *p, Eterm subject, BinaryFindState * return ret; } -static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState *bfs, - FindallData *fad, Uint fad_sz) +static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindContext **ctxp) { - size_t orig_size; + BinaryFindContext *ctx = (*ctxp); + BinaryFindAllContext *fa = &(ctx->u.fa); + FindallData *fad; Eterm orig; + size_t orig_size; Uint offset; Uint bit_offset; Uint bit_size; ErlSubBin *sb; + Uint do_trim; Sint i; - Sint tail; - Uint list_size; - Uint end_pos; - Uint do_trim = bfs->flags & (BINARY_SPLIT_TRIM | BINARY_SPLIT_TRIM_ALL); - Eterm *hp; - Eterm *hendp; - Eterm ret; + register Uint reds = ctx->reds; - tail = fad_sz - 1; - list_size = fad_sz + 1; - orig_size = binary_size(subject); - end_pos = (Uint)(orig_size); + if (ctx->state == BFSearch) { + if (ctx->pat_type == am_ac) { + fa->data = fa->d.ac.out; + fa->size = fa->d.ac.m; + } else { + fa->data = fa->d.bm.out; + fa->size = fa->d.bm.m; + } + fa->tail = fa->size - 1; + fa->head = fa->tail; + orig_size = binary_size(subject); + fa->end_pos = (Uint)(orig_size); + fa->term = NIL; + if (ctx->exported == 0 && ((fa->head + 1) >= reds)) { + ctx = bf_context_export(p, ctx); + *ctxp = ctx; + fa = &(ctx->u.fa); + } + erts_factory_proc_prealloc_init(&(fa->factory), p, (fa->size + 1) * (ERL_SUB_BIN_SIZE + 2)); + ctx->state = BFResult; + } - hp = HAlloc(p, list_size * (ERL_SUB_BIN_SIZE + 2)); - hendp = hp + list_size * (ERL_SUB_BIN_SIZE + 2); ERTS_GET_REAL_BIN(subject, orig, offset, bit_offset, bit_size); ASSERT(bit_size == 0); + fad = fa->data; + do_trim = ctx->flags & (BF_FLAG_SPLIT_TRIM | BF_FLAG_SPLIT_TRIM_ALL); - ret = NIL; - - for (i = tail; i >= 0; --i) { - sb = (ErlSubBin *)(hp); - sb->size = end_pos - (fad[i].pos + fad[i].len); + for (i = fa->head; i >= 0; --i) { + if (--reds == 0) { + ASSERT(ctx->exported == 1); + fa->head = i; + ctx->reds = reds; + if (!do_trim && (ctx->flags & BF_FLAG_SPLIT_TRIM)) { + ctx->flags &= ~BF_FLAG_SPLIT_TRIM; + } + return THE_NON_VALUE; + } + sb = (ErlSubBin *)(fa->factory.hp); + sb->size = fa->end_pos - (fad[i].pos + fad[i].len); if (!(sb->size == 0 && do_trim)) { sb->thing_word = HEADER_SUB_BIN; sb->offs = offset + fad[i].pos + fad[i].len; @@ -1738,15 +1817,18 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState * sb->bitoffs = bit_offset; sb->bitsize = 0; sb->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; - ret = CONS(hp, make_binary(sb), ret); - hp += 2; - do_trim &= ~BINARY_SPLIT_TRIM; + fa->factory.hp += ERL_SUB_BIN_SIZE; + fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term); + fa->factory.hp += 2; + do_trim &= ~BF_FLAG_SPLIT_TRIM; } - end_pos = fad[i].pos; + fa->end_pos = fad[i].pos; } - sb = (ErlSubBin *)(hp); + fa->head = i; + ctx->reds = reds; + + sb = (ErlSubBin *)(fa->factory.hp); sb->size = fad[0].pos; if (!(sb->size == 0 && do_trim)) { sb->thing_word = HEADER_SUB_BIN; @@ -1755,26 +1837,31 @@ static Eterm do_split_global_result(Process *p, Eterm subject, BinaryFindState * sb->bitoffs = bit_offset; sb->bitsize = 0; sb->is_writable = 0; - hp += ERL_SUB_BIN_SIZE; - ret = CONS(hp, make_binary(sb), ret); - hp += 2; + fa->factory.hp += ERL_SUB_BIN_SIZE; + fa->term = CONS(fa->factory.hp, make_binary(sb), fa->term); + fa->factory.hp += 2; } - HRelease(p, hendp, hp); - return ret; + erts_factory_close(&(fa->factory)); + + return fa->term; } static BIF_RETTYPE binary_find_trap(BIF_ALIST_3) { int runres; Eterm result; - Binary *bin = erts_magic_ref2bin(BIF_ARG_3); - - runres = do_binary_find(BIF_P, BIF_ARG_1, NULL, bin, BIF_ARG_2, &result); - if (runres == DO_BIN_MATCH_OK) { + Binary *ctx_bin = erts_magic_ref2bin(BIF_ARG_2); + Binary *pat_bin = erts_magic_ref2bin(BIF_ARG_3); + BinaryFindContext *ctx = NULL; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == bf_context_destructor); + runres = do_binary_find(BIF_P, BIF_ARG_1, &ctx, pat_bin, ctx_bin, &result); + if (runres == BF_OK) { + ASSERT(result != THE_NON_VALUE); BIF_RET(result); } else { - BUMP_ALL_REDS(BIF_P); - BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, result, BIF_ARG_3); + ASSERT(result == THE_NON_VALUE && ctx->trap_term != result && ctx->pat_term != result); + BIF_TRAP3(&binary_find_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } } diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index e9bfb39035..f673ef3194 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -50,13 +50,6 @@ #include "dtrace-wrapper.h" #include "lttng-wrapper.h" -#ifdef ERTS_SMP -#define DDLL_SMP 1 -#else -#define DDLL_SMP 0 -#endif - - /* * Local types */ @@ -107,18 +100,18 @@ static void dereference_all_processes(DE_Handle *dh); static void restore_process_references(DE_Handle *dh); static void ddll_no_more_references(void *vdh); -#define lock_drv_list() erts_smp_rwmtx_rwlock(&erts_driver_list_lock) -#define unlock_drv_list() erts_smp_rwmtx_rwunlock(&erts_driver_list_lock) +#define lock_drv_list() erts_rwmtx_rwlock(&erts_driver_list_lock) +#define unlock_drv_list() erts_rwmtx_rwunlock(&erts_driver_list_lock) #define assert_drv_list_locked() \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ - || erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ + || erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) #define assert_drv_list_rwlocked() \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock)) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock)) #define assert_drv_list_rlocked() \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) #define assert_drv_list_not_locked() \ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ - && !erts_smp_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) + ERTS_LC_ASSERT(!erts_lc_rwmtx_is_rwlocked(&erts_driver_list_lock) \ + && !erts_lc_rwmtx_is_rlocked(&erts_driver_list_lock)) #define FREE_PORT_FLAGS (ERTS_PORT_SFLGS_DEAD & (~ERTS_PORT_SFLG_INITIALIZING)) @@ -134,13 +127,13 @@ kill_ports_driver_unloaded(DE_Handle *dh) if (!prt) continue; - ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; state = erts_atomic32_read_nob(&prt->state); if (state & FREE_PORT_FLAGS) continue; - erts_smp_port_lock(prt); + erts_port_lock(prt); state = erts_atomic32_read_nob(&prt->state); if (!(state & ERTS_PORT_SFLGS_DEAD) && prt->drv_ptr->handle == dh) @@ -280,10 +273,8 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) path[path_len++] = '/'; sys_strcpy(path+path_len,name); -#if DDLL_SMP - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); -#endif if ((drv = lookup_driver(name)) != NULL) { if (drv->handle == NULL) { /* static_driver */ @@ -404,24 +395,18 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) erts_ddll_reference_driver(dh); ASSERT(dh->status == ERL_DE_RELOAD); dh->status = ERL_DE_FORCE_RELOAD; -#if DDLL_SMP unlock_drv_list(); -#endif kill_ports_driver_unloaded(dh); /* Dereference, eventually causing driver destruction */ -#if DDLL_SMP lock_drv_list(); -#endif erts_ddll_dereference_driver(dh); } -#if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); -#endif BIF_P->flags |= F_USING_DDLL; if (monitor) { @@ -432,18 +417,14 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_ok, ok_term); } -#if DDLL_SMP unlock_drv_list(); -#endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); soft_error: -#if DDLL_SMP unlock_drv_list(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); -#endif + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); if (do_build_load_error) { soft_error_term = build_load_error(BIF_P, build_this_load_error); } @@ -452,11 +433,11 @@ BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3) t = TUPLE2(hp, am_error, soft_error_term); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); BIF_RET(t); error: assert_drv_list_not_locked(); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); if (path != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path); } @@ -518,7 +499,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2) Eterm l; int kill_ports = 0; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); for(l = options; is_list(l); l = CDR(list_val(l))) { Eterm opt = CAR(list_val(l)); @@ -551,9 +532,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2) goto error; } -#if DDLL_SMP lock_drv_list(); -#endif if ((drv = lookup_driver(name)) == NULL) { soft_error_term = am_not_loaded; @@ -597,7 +576,7 @@ Eterm erl_ddll_try_unload_2(BIF_ALIST_2) dh->reload_full_path = dh->reload_driver_name = NULL; dh->reload_flags = 0; } - if (erts_smp_atomic32_read_nob(&dh->port_count) > 0) { + if (erts_atomic32_read_nob(&dh->port_count) > 0) { ++kill_ports; } dh->status = ERL_DE_UNLOAD; @@ -608,23 +587,17 @@ done: /* Avoid closing the driver by referencing it */ erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; -#if DDLL_SMP unlock_drv_list(); -#endif kill_ports_driver_unloaded(dh); -#if DDLL_SMP lock_drv_list(); -#endif erts_ddll_dereference_driver(dh); } -#if DDLL_SMP erts_ddll_reference_driver(dh); unlock_drv_list(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); lock_drv_list(); erts_ddll_dereference_driver(dh); -#endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); BIF_P->flags |= F_USING_DDLL; if (monitor > 0) { @@ -638,17 +611,13 @@ done: if (kill_ports > 1) { ERTS_BIF_CHK_EXITED(BIF_P); /* May be exited by port killing */ } -#if DDLL_SMP unlock_drv_list(); -#endif BIF_RET(t); soft_error: -#if DDLL_SMP unlock_drv_list(); -#endif erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); hp = HAlloc(BIF_P, 3); t = TUPLE2(hp, am_error, soft_error_term); BIF_RET(t); @@ -658,7 +627,7 @@ soft_error: if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_ERROR(BIF_P, BADARG); } @@ -697,9 +666,7 @@ BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0) int need = 3; Eterm res = NIL; erts_driver_t *drv; -#if DDLL_SMP lock_drv_list(); -#endif for (drv = driver_list; drv; drv = drv->next) { need += sys_strlen(drv->name)*2+2; } @@ -712,9 +679,7 @@ BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0) } res = TUPLE2(hp,am_ok,res); /* hp += 3 */ -#if DDLL_SMP unlock_drv_list(); -#endif BIF_RET(res); } @@ -736,9 +701,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) Eterm *hp; int i; Uint filter; -#if DDLL_SMP int have_lock = 0; -#endif if ((name = pick_list_or_atom(name_term)) == NULL) { goto error; @@ -748,10 +711,8 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) goto error; } -#if DDLL_SMP lock_drv_list(); have_lock = 1; -#endif if ((drv = lookup_driver(name)) == NULL) { goto error; } @@ -781,7 +742,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) } else if (drv->handle->status == ERL_DE_PERMANENT) { res = am_permanent; } else { - res = make_small(erts_smp_atomic32_read_nob(&drv->handle->port_count)); + res = make_small(erts_atomic32_read_nob(&drv->handle->port_count)); } goto done; case am_linked_in_driver: @@ -827,9 +788,7 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) hp += 2; } done: -#if DDLL_SMP unlock_drv_list(); -#endif if (pei) erts_free(ERTS_ALC_T_DDLL_TMP_BUF, pei); erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); @@ -838,11 +797,9 @@ BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2) if (name != NULL) { erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name); } -#if DDLL_SMP if (have_lock) { unlock_drv_list(); } -#endif BIF_ERROR(p,BADARG); } @@ -899,13 +856,9 @@ BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1) if (errdesc_to_code(code_term,&errint) != 0) { goto error; } -#if DDLL_SMP lock_drv_list(); -#endif errstring = erts_ddll_error(errint); -#if DDLL_SMP unlock_drv_list(); -#endif break; } if (errstring == NULL) { @@ -968,7 +921,7 @@ Eterm erts_ddll_monitor_driver(Process *p, void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks) { erts_driver_t *drv; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); lock_drv_list(); drv = driver_list; while (drv != NULL) { @@ -993,7 +946,7 @@ void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks) } done: unlock_drv_list(); - erts_smp_proc_lock(p, plocks); + erts_proc_lock(p, plocks); } /* @@ -1002,7 +955,7 @@ void erts_ddll_remove_monitor(Process *p, Eterm ref, ErtsProcLocks plocks) void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) { erts_driver_t *drv; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); lock_drv_list(); drv = driver_list; while (drv != NULL) { @@ -1040,18 +993,14 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) dh->status = ERL_DE_UNLOAD; } if (!left - && erts_smp_atomic32_read_nob(&drv->handle->port_count) > 0) { + && erts_atomic32_read_nob(&drv->handle->port_count) > 0) { if (kill_ports) { DE_Handle *dh = drv->handle; erts_ddll_reference_driver(dh); dh->status = ERL_DE_FORCE_UNLOAD; -#if DDLL_SMP unlock_drv_list(); -#endif kill_ports_driver_unloaded(dh); -#if DDLL_SMP lock_drv_list(); /* Needed for future list operations */ -#endif drv = drv->next; /* before allowing destruction */ erts_ddll_dereference_driver(dh); } else { @@ -1065,7 +1014,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks) } } unlock_drv_list(); - erts_smp_proc_lock(p, plocks); + erts_proc_lock(p, plocks); } void erts_ddll_lock_driver(DE_Handle *dh, char *name) { @@ -1093,41 +1042,41 @@ void erts_ddll_lock_driver(DE_Handle *dh, char *name) void erts_ddll_increment_port_count(DE_Handle *dh) { assert_drv_list_locked(); - erts_smp_atomic32_inc_nob(&dh->port_count); + erts_atomic32_inc_nob(&dh->port_count); } void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); #ifdef DEBUG - ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); + ASSERT(erts_atomic32_dec_read_nob(&dh->port_count) >= 0); #else - erts_smp_atomic32_dec_nob(&dh->port_count); + erts_atomic32_dec_nob(&dh->port_count); #endif } static void first_ddll_reference(DE_Handle *dh) { assert_drv_list_rwlocked(); - erts_smp_refc_init(&(dh->refc),1); + erts_refc_init(&(dh->refc),1); } void erts_ddll_reference_driver(DE_Handle *dh) { assert_drv_list_locked(); - if (erts_smp_refc_inctest(&(dh->refc),1) == 1) { - erts_smp_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */ + if (erts_refc_inctest(&(dh->refc),1) == 1) { + erts_refc_inc(&(dh->refc),2); /* add a reference for the scheduled operation */ } } void erts_ddll_reference_referenced_driver(DE_Handle *dh) { - erts_smp_refc_inc(&(dh->refc),2); + erts_refc_inc(&(dh->refc),2); } void erts_ddll_dereference_driver(DE_Handle *dh) { - if (erts_smp_refc_dectest(&(dh->refc),0) == 0) { + if (erts_refc_dectest(&(dh->refc),0) == 0) { /* No lock here, but if the driver is referenced again, the scheduled deletion is added as a reference too, see above */ erts_schedule_misc_op(ddll_no_more_references, (void *) dh); @@ -1150,11 +1099,11 @@ static void restore_process_references(DE_Handle *dh) { DE_ProcEntry *p; assert_drv_list_rwlocked(); - ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0); + ASSERT(erts_refc_read(&(dh->refc),0) == 0); for(p = dh->procs;p != NULL; p = p->next) { if (p->awaiting_status == ERL_DE_PROC_LOADED) { ASSERT(p->flags & ERL_DE_FL_DEREFERENCED); - erts_smp_refc_inc(&(dh->refc),1); + erts_refc_inc(&(dh->refc),1); p->flags &= ~ERL_DE_FL_DEREFERENCED; } } @@ -1176,9 +1125,9 @@ static void ddll_no_more_references(void *vdh) lock_drv_list(); - x = erts_smp_refc_read(&(dh->refc),0); + x = erts_refc_read(&(dh->refc),0); if (x > 0) { - x = erts_smp_refc_dectest(&(dh->refc),0); /* delete the reference added for me */ + x = erts_refc_dectest(&(dh->refc),0); /* delete the reference added for me */ } @@ -1281,10 +1230,8 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro Eterm immediate_type = NIL; erts_driver_t *drv; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks); -#if DDLL_SMP + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks); lock_drv_list(); -#endif if ((drv = lookup_driver(name)) == NULL) { immediate_tag = am_unloaded; immediate_type = am_DOWN; @@ -1314,20 +1261,14 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro } p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD); -#if DDLL_SMP unlock_drv_list(); -#endif BIF_RET(r); immediate: r = erts_make_ref(p); -#if DDLL_SMP - erts_smp_proc_unlock(p, plocks); -#endif + erts_proc_unlock(p, plocks); notify_proc(p, r, name_term, immediate_type, immediate_tag, 0); -#if DDLL_SMP unlock_drv_list(); - erts_smp_proc_lock(p, plocks); -#endif + erts_proc_lock(p, plocks); BIF_RET(r); } @@ -1338,10 +1279,8 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP Eterm immediate_type = NIL; erts_driver_t *drv; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks); -#if DDLL_SMP + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & plocks); lock_drv_list(); -#endif if ((drv = lookup_driver(name)) == NULL) { immediate_tag = am_unloaded; immediate_type = am_DOWN; @@ -1355,20 +1294,14 @@ static Eterm notify_when_unloaded(Process *p, Eterm name_term, char *name, ErtsP p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, flag); -#if DDLL_SMP unlock_drv_list(); -#endif BIF_RET(r); immediate: r = erts_make_ref(p); -#if DDLL_SMP - erts_smp_proc_unlock(p, plocks); -#endif + erts_proc_unlock(p, plocks); notify_proc(p, r, name_term, immediate_type, immediate_tag, 0); -#if DDLL_SMP unlock_drv_list(); - erts_smp_proc_lock(p, plocks); -#endif + erts_proc_lock(p, plocks); BIF_RET(r); } @@ -1572,8 +1505,8 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) res = ERL_DE_LOAD_ERROR_BAD_NAME; goto error; } - erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); - erts_smp_atomic32_init_nob(&dh->port_count, 0); + erts_atomic_init_nob(&(dh->refc), (erts_aint_t) 0); + erts_atomic32_init_nob(&dh->port_count, 0); dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); dh->flags = 0; @@ -1644,8 +1577,8 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) dh->handle = NULL; dh->procs = NULL; - erts_smp_atomic32_init_nob(&dh->port_count, 0); - erts_smp_refc_init(&(dh->refc), (erts_aint_t) 0); + erts_atomic32_init_nob(&dh->port_count, 0); + erts_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; dh->reload_driver_name = NULL; @@ -1683,7 +1616,7 @@ static int reload_driver_entry(DE_Handle *dh) dh->reload_full_path = NULL; dh->reload_driver_name = NULL; - ASSERT(erts_smp_refc_read(&(dh->refc),0) == 0); + ASSERT(erts_refc_read(&(dh->refc),0) == 0); ASSERT(dh->full_path != NULL); erts_free(ERTS_ALC_T_DDLL_HANDLE, (void *) dh->full_path); dh->full_path = NULL; @@ -1714,7 +1647,7 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, ErtsMessage *mp; ErtsProcLocks rp_locks = 0; ErlOffHeap *ohp; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; assert_drv_list_rwlocked(); if (errcode != 0) { @@ -1740,8 +1673,8 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type, mess = TUPLE5(hp,type,r,am_driver,driver_name,tag); } erts_queue_message(proc, rp_locks, mp, mess, am_system); - erts_smp_proc_unlock(proc, rp_locks); - ERTS_SMP_CHK_NO_PROC_LOCKS; + erts_proc_unlock(proc, rp_locks); + ERTS_CHK_NO_PROC_LOCKS; } static void notify_all(DE_Handle *dh, char *name, Uint awaiting, Eterm type, Eterm tag) @@ -1813,7 +1746,7 @@ static Eterm build_load_error(Process *p, int code) { int need = load_error_need(code); Eterm *hp = NULL; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); if (need) { hp = HAlloc(p,need); } diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index e2773475b0..17c936f041 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -46,8 +46,10 @@ #include "erl_thr_progress.h" #include "erl_bif_unique.h" #include "erl_map.h" +#include "erl_check_io.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" +#include "erl_time.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -87,24 +89,15 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #ifdef ARCH_64 " [64-bit]" #endif -#ifdef ERTS_SMP " [smp:%beu:%beu]" -#endif -#ifdef USE_THREADS -#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) " [ds:%beu:%beu:%beu]" -#endif #if defined(ERTS_DIRTY_SCHEDULERS_TEST) " [dirty-schedulers-TEST]" #endif " [async-threads:%d]" -#endif #ifdef HIPE " [hipe]" #endif -#ifdef ERTS_ENABLE_KERNEL_POLL - " [kernel-poll:%s]" -#endif #ifdef ET_DEBUG #if ET_DEBUG " [type-assertions]" @@ -353,14 +346,12 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) char *rc_str = ""; char rc_buf[100]; char *ov = otp_version; -#ifdef ERTS_SMP Uint total, online, active; Uint dirty_cpu, dirty_cpu_onln, dirty_io; erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, NULL, &dirty_io, NULL); -#endif for (i = 0; i < sizeof(otp_version)-4; i++) { if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c') rc = atoi(&ov[i+3]); @@ -375,18 +366,9 @@ erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) } return erts_print(to, arg, erts_system_version, rc_str -#ifdef ERTS_SMP , total, online -#ifdef ERTS_DIRTY_SCHEDULERS , dirty_cpu, dirty_cpu_onln, dirty_io -#endif -#endif -#ifdef USE_THREADS , erts_async_max_threads -#endif -#ifdef ERTS_ENABLE_KERNEL_POLL - , erts_use_kernel_poll ? "true" : "false" -#endif ); } @@ -763,7 +745,6 @@ process_info_init(void) static ERTS_INLINE Process * pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks) { -#ifdef ERTS_SMP /* * If the main lock is needed, we use erts_pid2proc_not_running() * instead of erts_pid2proc() for two reasons: @@ -781,7 +762,6 @@ pi_pid2proc(Process *c_p, Eterm pid, ErtsProcLocks info_locks) return erts_pid2proc_not_running(c_p, ERTS_PROC_LOCK_MAIN, pid, info_locks); else -#endif return erts_pid2proc(c_p, ERTS_PROC_LOCK_MAIN, pid, info_locks); } @@ -899,13 +879,13 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, * is being inspected... */ ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + ERTS_MSGQ_MV_INQ2PRIVQ(rp); locks &= ~ERTS_PROC_LOCK_MSGQ; unlock_locks |= ERTS_PROC_LOCK_MSGQ; } if (unlock_locks) - erts_smp_proc_unlock(rp, unlock_locks); + erts_proc_unlock(rp, unlock_locks); } @@ -963,7 +943,7 @@ process_info_list(Process *c_p, Eterm pid, Eterm list, int always_wrap, if (c_p == rp) locks &= ~ERTS_PROC_LOCK_MAIN; if (locks && rp) - erts_smp_proc_unlock(rp, locks); + erts_proc_unlock(rp, locks); if (res_elem_ix != &def_res_elem_ix_buf[0]) erts_free(ERTS_ALC_T_TMP, res_elem_ix); @@ -1054,7 +1034,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) ERTS_BIF_YIELD2(bif_export[BIF_process_info_2], BIF_P, BIF_ARG_1, BIF_ARG_2); else if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) { - erts_smp_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, info_locks|ERTS_PROC_LOCK_STATUS); ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); } else { @@ -1074,24 +1054,22 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) * is being inspected... */ ASSERT(info_locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + ERTS_MSGQ_MV_INQ2PRIVQ(rp); info_locks &= ~ERTS_PROC_LOCK_MSGQ; unlock_locks |= ERTS_PROC_LOCK_MSGQ; } if (unlock_locks) - erts_smp_proc_unlock(rp, unlock_locks); + erts_proc_unlock(rp, unlock_locks); res = process_info_aux(BIF_P, rp, info_locks, pid, BIF_ARG_2, 0); } ASSERT(is_value(res)); -#ifdef ERTS_SMP if (BIF_P == rp) info_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp && info_locks) - erts_smp_proc_unlock(rp, info_locks); -#endif + erts_proc_unlock(rp, info_locks); ASSERT(!(BIF_P->flags & F_P2PNR_RESCHED)); BIF_RET(res); @@ -1378,7 +1356,7 @@ process_info_aux(Process *BIF_P, break; case am_trap_exit: { - erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); hp = HAlloc(BIF_P, 3); if (state & ERTS_PSFLG_TRAP_EXIT) res = am_true; @@ -2144,14 +2122,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ASSERT(erts_compat_rel > 0); BIF_RET(make_small(erts_compat_rel)); } else if (BIF_ARG_1 == am_multi_scheduling) { -#ifndef ERTS_SMP - BIF_RET(am_disabled); -#else -#ifndef ERTS_DIRTY_SCHEDULERS - if (erts_no_schedulers == 1) - BIF_RET(am_disabled); - else -#endif { int msb = erts_is_multi_scheduling_blocked(); BIF_RET(!msb @@ -2160,7 +2130,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ? am_blocked : am_blocked_normal)); } -#endif } else if (BIF_ARG_1 == am_build_type) { #if defined(DEBUG) ERTS_DECL_AM(debug); @@ -2270,7 +2239,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); + Uint val = (Uint) erts_atomic32_read_nob(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2); @@ -2288,7 +2257,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); + Uint val = (Uint) erts_atomic32_read_nob(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -2321,8 +2290,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_info_dsbuf(0); /* Need to be the only thread running... */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); if (BIF_ARG_1 == am_info) info(ERTS_PRINT_DSBUF, (void *) dsbufp); @@ -2333,8 +2302,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else distribution_info(ERTS_PRINT_DSBUF, (void *) dsbufp); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); @@ -2343,7 +2312,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { DistEntry *dep; i = 0; - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); for (dep = erts_visible_dist_entries; dep; dep = dep->next) ++i; for (dep = erts_hidden_dist_entries; dep; dep = dep->next) @@ -2366,7 +2335,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = CONS(hp, tpl, res); hp += 2; } - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); BIF_RET(res); } else if (BIF_ARG_1 == am_system_version) { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); @@ -2392,16 +2361,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_allocator_options((void *) BIF_P)); } else if (BIF_ARG_1 == am_thread_pool_size) { -#ifdef USE_THREADS extern int erts_async_max_threads; -#endif int n; -#ifdef USE_THREADS n = erts_async_max_threads; -#else - n = 0; -#endif BIF_RET(make_small(n)); } else if (BIF_ARG_1 == am_alloc_util_allocators) { @@ -2469,7 +2432,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #endif BIF_RET(res); -#endif /* #ifndef ERTS_SMP */ +#endif /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */ } else if (BIF_ARG_1 == am_wordsize) { return make_small(sizeof(Eterm)); } else if (BIF_ARG_1 == am_endian) { @@ -2549,11 +2512,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); #endif } else if (BIF_ARG_1 == am_threads) { -#ifdef USE_THREADS return am_true; -#else - return am_false; -#endif } else if (BIF_ARG_1 == am_creation) { return make_small(erts_this_node->creation); } else if (BIF_ARG_1 == am_break_ignored) { @@ -2612,11 +2571,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = HAlloc(BIF_P, 2*n); BIF_RET(buf_to_intlist(&hp, buf, n, NIL)); } else if (ERTS_IS_ATOM_STR("smp_support", BIF_ARG_1)) { -#ifdef ERTS_SMP BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) { BIF_RET(erts_bound_schedulers_term(BIF_P)); } else if (ERTS_IS_ATOM_STR("scheduler_bindings", BIF_ARG_1)) { @@ -2628,11 +2583,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = make_small(erts_no_schedulers); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) { -#ifndef ERTS_SMP - Eterm *hp = HAlloc(BIF_P, 4); - res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); - BIF_RET(res); -#else Eterm *hp; Uint total, online, active; erts_schedulers_state(&total, &online, &active, @@ -2643,13 +2593,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) make_small(online), make_small(active)); BIF_RET(res); -#endif } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) { -#ifndef ERTS_SMP - Eterm *hp = HAlloc(BIF_P, 4); - res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); - BIF_RET(res); -#else Eterm *hp; Uint total, online, active; erts_schedulers_state(&total, &online, &active, @@ -2660,19 +2604,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) make_small(online), make_small(active)); BIF_RET(res); -#endif } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) { -#ifndef ERTS_SMP - Eterm *hp = HAlloc(BIF_P, 2+5); - res = CONS(hp+5, - TUPLE4(hp, - am_normal, - make_small(1), - make_small(1), - make_small(1)), - NIL); - BIF_RET(res); -#else Eterm *hp, tpl; Uint sz, total, online, active, dirty_cpu_total, dirty_cpu_online, dirty_cpu_active, @@ -2718,46 +2650,25 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp += 5; res = CONS(hp, tpl, res); BIF_RET(res); -#endif } else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) { -#ifndef ERTS_SMP - BIF_RET(make_small(1)); -#else Uint online; erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL); BIF_RET(make_small(online)); -#endif } else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) { -#ifndef ERTS_SMP - BIF_RET(make_small(1)); -#else Uint active; erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL); BIF_RET(make_small(active)); -#endif } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); -#else - dirty_cpu = 0; -#endif BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); -#else - dirty_cpu_onln = 0; -#endif BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; -#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); -#else - dirty_io = 0; -#endif BIF_RET(make_small(dirty_io)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); @@ -2779,7 +2690,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(make_small(CONTEXT_REDS)); } else if (ERTS_IS_ATOM_STR("kernel_poll", BIF_ARG_1)) { #ifdef ERTS_ENABLE_KERNEL_POLL - BIF_RET(erts_use_kernel_poll ? am_true : am_false); + BIF_RET(am_true); #else BIF_RET(am_false); #endif @@ -2804,23 +2715,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("check_io", BIF_ARG_1)) { BIF_RET(erts_check_io_info(BIF_P)); } else if (ERTS_IS_ATOM_STR("multi_scheduling_blockers", BIF_ARG_1)) { -#ifndef ERTS_SMP - BIF_RET(NIL); -#else if (erts_no_schedulers == 1) BIF_RET(NIL); else BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0)); -#endif } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) { -#ifndef ERTS_SMP - BIF_RET(NIL); -#else if (erts_no_schedulers == 1) BIF_RET(NIL); else BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1)); -#endif } else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) { BIF_RET(ERTS_USE_MODIFIED_TIMING() ? make_small(erts_modified_timing_level) @@ -2883,12 +2786,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_false); #endif } -#ifdef ERTS_SMP else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) { erts_thr_progress_dbg_print_state(); BIF_RET(am_true); } -#endif else if (BIF_ARG_1 == am_message_queue_data) { switch (erts_default_spo_flags & (SPO_ON_HEAP_MSGQ|SPO_OFF_HEAP_MSGQ)) { case SPO_OFF_HEAP_MSGQ: @@ -2942,7 +2843,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_disabled); } else if (ERTS_IS_ATOM_STR("eager_check_io",BIF_ARG_1)) { - BIF_RET(erts_eager_check_io ? am_true : am_false); + BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("literal_test",BIF_ARG_1)) { #ifdef ERTS_HAVE_IS_IN_LITERAL_RANGE @@ -2980,7 +2881,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, { Eterm res = THE_NON_VALUE; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (item == am_id) { if (hpp) @@ -3171,9 +3072,6 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, } else if (ERTS_IS_ATOM_STR("locking", item)) { if (hpp) { -#ifndef ERTS_SMP - res = am_false; -#else if (erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { DECL_AM(port_level); @@ -3187,7 +3085,6 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, & ERL_DRV_FLAG_USE_PORT_LOCKING)); res = AM_driver_level; } -#endif } if (szp) { res = am_true; @@ -3200,7 +3097,7 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, goto done; } res = ((ERTS_PTS_FLG_PARALLELISM & - erts_smp_atomic32_read_nob(&prt->sched.flags)) + erts_atomic32_read_nob(&prt->sched.flags)) ? am_true : am_false); } @@ -3276,7 +3173,7 @@ fun_info_2(BIF_ALIST_2) } break; case am_refc: - val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p); + val = erts_make_integer(erts_atomic_read_nob(&funp->fe->refc), p); hp = HAlloc(p, 3); break; case am_arity: @@ -3381,7 +3278,7 @@ BIF_RETTYPE is_process_alive_1(BIF_ALIST_1) BIF_RET(am_false); } else { - if (erts_smp_atomic32_read_acqb(&rp->state) + if (erts_atomic32_read_acqb(&rp->state) & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_false); else @@ -3416,7 +3313,7 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) BIF_ARG_1, BIF_ARG_2); if (rp != BIF_P && ERTS_PROC_PENDING_EXIT(rp)) { Eterm args[2] = {BIF_ARG_1, BIF_ARG_2}; - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); + erts_proc_unlock(rp, ERTS_PROC_LOCKS_ALL); ERTS_BIF_AWAIT_X_APPLY_TRAP(BIF_P, BIF_ARG_1, am_erlang, @@ -3425,11 +3322,9 @@ BIF_RETTYPE process_display_2(BIF_ALIST_2) 2); } erts_stack_dump(ERTS_PRINT_STDERR, NULL, rp); -#ifdef ERTS_SMP - erts_smp_proc_unlock(rp, (BIF_P == rp + erts_proc_unlock(rp, (BIF_P == rp ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); -#endif BIF_RET(am_true); } @@ -3544,24 +3439,32 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_runtime) { - UWord u1, u2, dummy; + ErtsMonotonicTime u1, u2; Eterm b1, b2; - elapsed_time_both(&u1,&dummy,&u2,&dummy); - b1 = erts_make_integer(u1,BIF_P); - b2 = erts_make_integer(u2,BIF_P); - hp = HAlloc(BIF_P,3); + Uint hsz; + erts_runtime_elapsed_both(&u1, NULL, &u2, NULL); + hsz = 3; /* 2-tuple */ + (void) erts_bld_monotonic_time(NULL, &hsz, u1); + (void) erts_bld_monotonic_time(NULL, &hsz, u2); + hp = HAlloc(BIF_P, hsz); + b1 = erts_bld_monotonic_time(&hp, NULL, u1); + b2 = erts_bld_monotonic_time(&hp, NULL, u2); res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_run_queue) { res = erts_run_queues_len(NULL, 1, 0, 0); BIF_RET(make_small(res)); } else if (BIF_ARG_1 == am_wall_clock) { - UWord w1, w2; + ErtsMonotonicTime w1, w2; Eterm b1, b2; - wall_clock_elapsed_time_both(&w1, &w2); - b1 = erts_make_integer((Uint) w1,BIF_P); - b2 = erts_make_integer((Uint) w2,BIF_P); - hp = HAlloc(BIF_P,3); + Uint hsz; + erts_wall_clock_elapsed_both(&w1, &w2); + hsz = 3; /* 2-tuple */ + (void) erts_bld_monotonic_time(NULL, &hsz, w1); + (void) erts_bld_monotonic_time(NULL, &hsz, w2); + hp = HAlloc(BIF_P, hsz); + b1 = erts_bld_monotonic_time(&hp, NULL, w1); + b2 = erts_bld_monotonic_time(&hp, NULL, w2); res = TUPLE2(hp, b1, b2); BIF_RET(res); } else if (BIF_ARG_1 == am_io) { @@ -3599,7 +3502,7 @@ BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0) BIF_RET(erts_error_logger_warnings); } -static erts_smp_atomic_t available_internal_state; +static erts_atomic_t available_internal_state; static int empty_magic_ref_destructor(Binary *bin) { @@ -3612,7 +3515,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) * NOTE: Only supposed to be used for testing, and debugging. */ - if (!erts_smp_atomic_read_nob(&available_internal_state)) { + if (!erts_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -3655,9 +3558,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) int no_errors; ErtsCheckIoDebugInfo ciodi = {0}; #ifdef HAVE_ERTS_CHECK_IO_DEBUG - erts_smp_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P,ERTS_PROC_LOCK_MAIN); no_errors = erts_check_io_debug(&ciodi); - erts_smp_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P,ERTS_PROC_LOCK_MAIN); #else no_errors = 0; #endif @@ -3672,8 +3575,8 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) (Uint) ciodi.no_used_fds), erts_bld_uint(hpp, szp, (Uint) ciodi.no_driver_select_structs), - erts_bld_uint(hpp, szp, - (Uint) ciodi.no_driver_event_structs)); + erts_bld_uint(hpp, szp, + (Uint) ciodi.no_enif_select_structs)); if (hpp) break; hp = HAlloc(BIF_P, sz); @@ -3707,9 +3610,9 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("nbalance", BIF_ARG_1)) { Uint n; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); n = erts_debug_nbalance(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(erts_make_integer(n, BIF_P)); } else if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)) { @@ -3724,11 +3627,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) { Eterm res; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE); - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("mmap", BIF_ARG_1)) { @@ -3795,11 +3698,11 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) tp[2], ERTS_PROC_LOCK_LINK); if (!p) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); + ERTS_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } res = make_link_list(BIF_P, ERTS_P_LINKS(p), NIL); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); BIF_RET(res); } else if(is_internal_port(tp[2])) { @@ -3818,11 +3721,10 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) DistEntry *dep = erts_find_dist_entry(tp[2]); if(dep) { Eterm subres; - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); subres = make_link_list(BIF_P, dep->nlinks, NIL); subres = make_link_list(BIF_P, dep->node_links, subres); - erts_smp_de_links_unlock(dep); - erts_deref_dist_entry(dep); + erts_de_links_unlock(dep); BIF_RET(subres); } else { BIF_RET(am_undefined); @@ -3840,20 +3742,19 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) tp[2], ERTS_PROC_LOCK_LINK); if (!p) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(BIF_P); + ERTS_ASSERT_IS_NOT_EXITING(BIF_P); BIF_RET(am_undefined); } res = make_monitor_list(BIF_P, ERTS_P_MONITORS(p)); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(p, ERTS_PROC_LOCK_LINK); BIF_RET(res); } else if(is_node_name_atom(tp[2])) { DistEntry *dep = erts_find_dist_entry(tp[2]); if(dep) { Eterm ml; - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); ml = make_monitor_list(BIF_P, dep->monitors); - erts_smp_de_links_unlock(dep); - erts_deref_dist_entry(dep); + erts_de_links_unlock(dep); BIF_RET(ml); } else { BIF_RET(am_undefined); @@ -3868,7 +3769,6 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else { Uint cno = dist_entry_channel_no(dep); res = make_small(cno); - erts_deref_dist_entry(dep); } BIF_RET(res); } @@ -3880,7 +3780,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) } else { Eterm res = ERTS_PROC_PENDING_EXIT(rp) ? am_true : am_false; - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); BIF_RET(res); } } @@ -3930,15 +3830,14 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) DFLAG_BIT_BINARIES); BIF_RET(erts_term_to_binary(BIF_P, tp[2], 0, dflags)); } - else if (ERTS_IS_ATOM_STR("dist_port", tp[1])) { + else if (ERTS_IS_ATOM_STR("dist_ctrl", tp[1])) { Eterm res = am_undefined; DistEntry *dep = erts_sysname_to_connected_dist_entry(tp[2]); if (dep) { - erts_smp_de_rlock(dep); - if (is_internal_port(dep->cid)) + erts_de_rlock(dep); + if (is_internal_port(dep->cid) || is_internal_pid(dep->cid)) res = dep->cid; - erts_smp_de_runlock(dep); - erts_deref_dist_entry(dep); + erts_de_runlock(dep); } BIF_RET(res); } @@ -4082,7 +3981,7 @@ BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } -static erts_smp_atomic_t hipe_test_reschedule_flag; +static erts_atomic_t hipe_test_reschedule_flag; #if defined(VALGRIND) && defined(__GNUC__) /* Force noinline for valgrind suppression */ @@ -4106,7 +4005,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); - erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on); + erts_aint_t prev_on = erts_atomic_xchg_nob(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->common.id); @@ -4122,7 +4021,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(prev_on ? am_true : am_false); } - if (!erts_smp_atomic_read_nob(&available_internal_state)) { + if (!erts_atomic_read_nob(&available_internal_state)) { BIF_ERROR(BIF_P, EXC_UNDEF); } @@ -4146,13 +4045,13 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) Sint ms; if (term_to_Sint(BIF_ARG_2, &ms) != 0) { if (ms > 0) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); if (block) - erts_smp_thr_progress_block(); + erts_thr_progress_block(); while (erts_milli_sleep((long) ms) != 0); if (block) - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); } @@ -4161,9 +4060,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) Sint ms; if (term_to_Sint(BIF_ARG_2, &ms) != 0) { if (ms > 0) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); while (erts_milli_sleep((long) ms) != 0); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } BIF_RET(am_true); } @@ -4231,10 +4130,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(AM_dead); } -#ifdef ERTS_SMP if (BIF_P == rp) rp_locks |= ERTS_PROC_LOCK_MAIN; -#endif xres = erts_send_exit_signal(NULL, /* NULL in order to force a pending exit when we send to our @@ -4246,11 +4143,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) NIL, NULL, 0); -#ifdef ERTS_SMP if (BIF_P == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; -#endif - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (xres > 1) { DECL_AM(message); BIF_RET(AM_message); @@ -4312,14 +4207,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_atomic_read_nob(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { - erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1); + erts_atomic_set_nob(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2], BIF_P, BIF_ARG_1, BIF_ARG_2); } - erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag); + erts_atomic_set_nob(&hipe_test_reschedule_flag, !flag); BIF_RET(NIL); } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) { @@ -4330,7 +4225,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) if (rp) { erts_resume(rp, ERTS_PROC_LOCK_STATUS); res = am_true; - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); } BIF_RET(res); } @@ -4347,16 +4242,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_false); else { Uint32 con_id; - erts_smp_de_rlock(dep); + erts_de_rlock(dep); con_id = dep->connection_id; - erts_smp_de_runlock(dep); + erts_de_runlock(dep); erts_kill_dist_connection(dep, con_id); - erts_deref_dist_entry(dep); BIF_RET(am_true); } } else if (ERTS_IS_ATOM_STR("not_running_optimization", BIF_ARG_1)) { -#ifdef ERTS_SMP int old_use_opt, use_opt; switch (BIF_ARG_2) { case am_true: @@ -4369,16 +4262,13 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); old_use_opt = !erts_disable_proc_not_running_opt; erts_disable_proc_not_running_opt = !use_opt; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(old_use_opt ? am_true : am_false); -#else - BIF_ERROR(BIF_P, EXC_NOTSUP); -#endif } else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) { if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) { @@ -4406,9 +4296,9 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) Sint64 msecs; if (term_to_Sint64(BIF_ARG_2, &msecs)) { /* Negative value restore original value... */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_debug_test_node_tab_delayed_delete(msecs); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(am_ok); } } @@ -4444,48 +4334,120 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } #ifdef ERTS_ENABLE_LOCK_COUNT + +typedef struct { + /* info->location_count may increase between size calculation and term + * building, so we cap it at the value sampled in lcnt_build_result_vector. + * + * Shrinking is safe though. */ + int max_location_count; + erts_lcnt_lock_info_t *info; +} lcnt_sample_t; + +typedef struct lcnt_sample_vector_ { + lcnt_sample_t *elements; + size_t size; +} lcnt_sample_vector_t; + +static lcnt_sample_vector_t lcnt_build_sample_vector(erts_lcnt_lock_info_list_t *list) { + erts_lcnt_lock_info_t *iterator; + lcnt_sample_vector_t result; + size_t allocated_entries; + + allocated_entries = 64; + result.size = 0; + + result.elements = erts_alloc(ERTS_ALC_T_LCNT_VECTOR, + allocated_entries * sizeof(lcnt_sample_t)); + + iterator = NULL; + while(erts_lcnt_iterate_list(list, &iterator)) { + erts_lcnt_retain_lock_info(iterator); + + result.elements[result.size].max_location_count = iterator->location_count; + result.elements[result.size].info = iterator; + + result.size++; + + if(result.size >= allocated_entries) { + allocated_entries *= 2; + + result.elements = erts_realloc(ERTS_ALC_T_LCNT_VECTOR, result.elements, + allocated_entries * sizeof(lcnt_sample_t)); + } + } + + return result; +} + +static void lcnt_destroy_sample_vector(lcnt_sample_vector_t *vector) { + size_t i; + + for(i = 0; i < vector->size; i++) { + erts_lcnt_release_lock_info(vector->elements[i].info); + } + + erts_free(ERTS_ALC_T_LCNT_VECTOR, vector->elements); +} + +/* The size of an integer is not guaranteed to be constant since we're walking + * over live data, and may cross over into bignum territory between size calc + * and the actual build. This takes care of that through always assuming the + * worst, but needs to be fixed up with HRelease once the final term has been + * built. */ +static ERTS_INLINE Eterm bld_unstable_uint64(Uint **hpp, Uint *szp, Uint64 ui) { + Eterm res = THE_NON_VALUE; + + if(szp) { + *szp += ERTS_UINT64_HEAP_SIZE(~((Uint64) 0)); + } + + if(hpp) { + if (IS_USMALL(0, ui)) { + res = make_small(ui); + } else { + res = erts_uint64_to_big(ui, hpp); + } + } + + return res; +} + static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) { - Uint tries = 0, colls = 0; - unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; - unsigned int line = 0; unsigned int i; - + const char *file; + Eterm af, uil; Eterm uit, uic; Eterm uits, uitns, uitn; Eterm tt, tstat, tloc, t; Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; - + /* term: - * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}, - * { .. histogram .. }] - */ + * [{{file, line}, + {tries, colls, {seconds, nanoseconds, n_blocks}}, + * { .. histogram .. }] */ - tries = (Uint) ethr_atomic_read(&stats->tries); - colls = (Uint) ethr_atomic_read(&stats->colls); - - line = stats->line; - timer_s = stats->timer.s; - timer_ns = stats->timer.ns; - timer_n = stats->timer_n; - - af = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); - uil = erts_bld_uint( hpp, szp, line); + file = stats->file ? stats->file : "undefined"; + + af = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1); + uil = erts_bld_uint( hpp, szp, stats->line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); - - uit = erts_bld_uint( hpp, szp, tries); - uic = erts_bld_uint( hpp, szp, colls); - uits = erts_bld_uint( hpp, szp, timer_s); - uitns = erts_bld_uint( hpp, szp, timer_ns); - uitn = erts_bld_uint( hpp, szp, timer_n); + uit = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->attempts)); + uic = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->collisions)); + + uits = bld_unstable_uint64(hpp, szp, stats->total_time_waited.s); + uitns = bld_unstable_uint64(hpp, szp, stats->total_time_waited.ns); + uitn = bld_unstable_uint64(hpp, szp, stats->times_waited); tt = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn); tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt); for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) { - vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]); + vhist[i] = bld_unstable_uint64(hpp, szp, stats->wait_time_histogram.ns[i]); } + thist = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist); t = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist); @@ -4494,185 +4456,266 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s return res; } -static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock, Eterm res) { +static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) { + Eterm id = info->id; + + if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_TYPE_PROCLOCK) { + /* Use registered names as id's for process locks if available. Thread + * progress is delayed since we may be running on a dirty scheduler. */ + ErtsThrPrgrDelayHandle delay_handle; + Process *process; + + delay_handle = erts_thr_progress_unmanaged_delay(); + + process = erts_proc_lookup(info->id); + if (process && process->common.u.alive.reg) { + id = process->common.u.alive.reg->name; + } + + erts_thr_progress_unmanaged_continue(delay_handle); + } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) { + if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) { + const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id)); + id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1); + } + } + + return id; +} + +static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample, Eterm res) { + erts_lcnt_lock_info_t *info = sample->info; + Eterm name, type, id, stats = NIL, t; - Process *proc = NULL; - char *ltype; + const char *lock_desc; int i; + + /* term: [{name, id, type, stats()}] */ + + ASSERT(info->name); - /* term: - * [{name, id, type, stats()}] - */ - - ASSERT(lock->name); - - ltype = erts_lcnt_lock_type(lock->flag); - - ASSERT(ltype); - - type = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); - - if (lock->flag & ERTS_LCNT_LT_ALLOC) { - /* use allocator types names as id's for allocator locks */ - ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id)); - id = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { - /* use registered names as id's for process locks if available */ - proc = erts_proc_lookup(lock->id); - if (proc && proc->common.u.alive.reg) { - id = proc->common.u.alive.reg->name; - } else { - /* otherwise use process id */ - id = lock->id; - } + lock_desc = erts_lock_flags_get_type_name(info->flags); + + type = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); + + /* Only attempt to resolve ids when actually emitting the term. This ought + * to be safe since all immediates are the same size. */ + if(hpp != NULL) { + id = lcnt_pretty_print_lock_id(info); } else { - id = lock->id; + id = NIL; } - for (i = 0; i < lock->n_stats; i++) { - stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats); + for(i = 0; i < MIN(info->location_count, sample->max_location_count); i++) { + stats = lcnt_build_lock_stats_term(hpp, szp, &(info->location_stats[i]), stats); } t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); - res = erts_bld_cons( hpp, szp, t, res); + res = erts_bld_cons(hpp, szp, t, res); return res; } -static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *data, Eterm res) { +static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *duration, + lcnt_sample_vector_t *current_locks, + lcnt_sample_vector_t *deleted_locks, Eterm res) { + const char *str_duration = "duration"; + const char *str_locks = "locks"; + Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc; - erts_lcnt_lock_t *lock = NULL; - char *str_duration = "duration"; - char *str_locks = "locks"; - - /* term: - * [{'duration', {seconds, nanoseconds}}, {'locks', locks()}] - */ - + size_t i; + + /* term: [{'duration', {seconds, nanoseconds}}, {'locks', locks()}] */ + /* duration tuple */ - dts = erts_bld_uint( hpp, szp, data->duration.s); - dtns = erts_bld_uint( hpp, szp, data->duration.ns); + dts = bld_unstable_uint64(hpp, szp, duration->s); + dtns = bld_unstable_uint64(hpp, szp, duration->ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - + adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ - aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); - - for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) { - lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); + + for(i = 0; i < current_locks->size; i++) { + lloc = lcnt_build_lock_term(hpp, szp, ¤t_locks->elements[i], lloc); } - - for (lock = data->deleted_locks->head; lock != NULL ; lock = lock->next ) { - lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); + + for(i = 0; i < deleted_locks->size; i++) { + lloc = lcnt_build_lock_term(hpp, szp, &deleted_locks->elements[i], lloc); } - + tloc = erts_bld_tuple(hpp, szp, 2, aloc, lloc); - - res = erts_bld_cons( hpp, szp, tloc, res); - res = erts_bld_cons( hpp, szp, tdur, res); + + res = erts_bld_cons(hpp, szp, tloc, res); + res = erts_bld_cons(hpp, szp, tdur, res); + + return res; +} + +static struct { + const char *name; + erts_lock_flags_t flag; +} lcnt_category_map[] = { + {"allocator", ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR}, + {"db", ERTS_LOCK_FLAGS_CATEGORY_DB}, + {"debug", ERTS_LOCK_FLAGS_CATEGORY_DEBUG}, + {"distribution", ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION}, + {"generic", ERTS_LOCK_FLAGS_CATEGORY_GENERIC}, + {"io", ERTS_LOCK_FLAGS_CATEGORY_IO}, + {"process", ERTS_LOCK_FLAGS_CATEGORY_PROCESS}, + {"scheduler", ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER}, + {NULL, 0} + }; + +static erts_lock_flags_t lcnt_atom_to_lock_category(Eterm atom) { + int i = 0; + + for(i = 0; lcnt_category_map[i].name != NULL; i++) { + if(erts_is_atom_str(lcnt_category_map[i].name, atom, 0)) { + return lcnt_category_map[i].flag; + } + } + + return 0; +} + +static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t mask) { + Eterm res; + int i; + + res = NIL; + + for(i = 0; lcnt_category_map[i].name != NULL; i++) { + if(mask & lcnt_category_map[i].flag) { + Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name, + strlen(lcnt_category_map[i].name), + ERTS_ATOM_ENC_UTF8, 0); + + res = erts_bld_cons(hpp, szp, category, res); + } + } return res; -} +} + #endif -BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) +BIF_RETTYPE erts_debug_lcnt_clear_0(BIF_ALIST_0) { -#ifdef ERTS_ENABLE_LOCK_COUNT - Eterm res = NIL; -#endif +#ifndef ERTS_ENABLE_LOCK_COUNT + BIF_RET(am_error); +#else + erts_lcnt_clear_counters(); + BIF_RET(am_ok); +#endif +} - if (BIF_ARG_1 == am_enabled) { -#ifdef ERTS_ENABLE_LOCK_COUNT - BIF_RET(am_true); +BIF_RETTYPE erts_debug_lcnt_collect_0(BIF_ALIST_0) +{ +#ifndef ERTS_ENABLE_LOCK_COUNT + BIF_RET(am_error); #else - BIF_RET(am_false); + lcnt_sample_vector_t current_locks, deleted_locks; + erts_lcnt_data_t data; + + Eterm *term_heap_start, *term_heap_end; + Uint term_heap_size = 0; + Eterm result; + + data = erts_lcnt_get_data(); + + current_locks = lcnt_build_sample_vector(data.current_locks); + deleted_locks = lcnt_build_sample_vector(data.deleted_locks); + + lcnt_build_result_term(NULL, &term_heap_size, &data.duration, + ¤t_locks, &deleted_locks, NIL); + + term_heap_start = HAlloc(BIF_P, term_heap_size); + term_heap_end = term_heap_start; + + result = lcnt_build_result_term(&term_heap_end, NULL, + &data.duration, ¤t_locks, &deleted_locks, NIL); + + HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end); + + lcnt_destroy_sample_vector(¤t_locks); + lcnt_destroy_sample_vector(&deleted_locks); + + BIF_RET(result); #endif - } +} + +BIF_RETTYPE erts_debug_lcnt_control_1(BIF_ALIST_1) +{ #ifdef ERTS_ENABLE_LOCK_COUNT + if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) { + erts_lock_flags_t mask; + Eterm *term_heap_block; + Uint term_heap_size; - else if (BIF_ARG_1 == am_info) { - erts_lcnt_data_t *data; - Uint hsize = 0; - Uint *szp; - Eterm* hp; + mask = erts_lcnt_get_category_mask(); + term_heap_size = 0; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + lcnt_build_category_list(NULL, &term_heap_size, mask); - erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND); - data = erts_lcnt_get_data(); + term_heap_block = HAlloc(BIF_P, term_heap_size); - /* calculate size */ + BIF_RET(lcnt_build_category_list(&term_heap_block, NULL, mask)); + } else if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) { + if(erts_lcnt_get_preserve_info()) { + BIF_RET(am_true); + } - szp = &hsize; - lcnt_build_result_term(NULL, szp, data, NIL); + BIF_RET(am_false); + } +#endif + BIF_ERROR(BIF_P, BADARG); +} - /* alloc and build */ +BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2) +{ +#ifdef ERTS_ENABLE_LOCK_COUNT + if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) { + erts_lock_flags_t category_mask = 0; + Eterm categories = BIF_ARG_2; - hp = HAlloc(BIF_P, hsize); + if(!(is_list(categories) || is_nil(categories))) { + BIF_ERROR(BIF_P, BADARG); + } - res = lcnt_build_result_term(&hp, NULL, data, res); - - erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND); + while(is_list(categories)) { + Eterm *cell = list_val(categories); + erts_lock_flags_t category; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - - BIF_RET(res); - } else if (BIF_ARG_1 == am_clear) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - - erts_lcnt_clear_counters(); - - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - - BIF_RET(am_ok); - } else if (is_tuple(BIF_ARG_1)) { - Eterm* ptr = tuple_val(BIF_ARG_1); - - if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) { - int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0; - if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_COPYSAVE; - } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_PROCLOCK; - } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_PORTLOCK; - } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_SUSPEND; - } else if (ERTS_IS_ATOM_STR("location", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_LOCATION; - } else { - BIF_ERROR(BIF_P, BADARG); - } + category = lcnt_atom_to_lock_category(CAR(cell)); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - - if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; - else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; - -#ifdef ERTS_SMP - if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) { - erts_lcnt_enable_io_lock_count(enable); - } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) { - erts_lcnt_enable_proc_lock_count(enable); + if(!category) { + Eterm *hp = HAlloc(BIF_P, 4); + + BIF_RET(TUPLE3(hp, am_error, am_badarg, CAR(cell))); } -#endif - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - BIF_RET(res); + + category_mask |= category; + categories = CDR(cell); } - } -#endif + erts_lcnt_set_category_mask(category_mask); + + BIF_RET(am_ok); + } else if(BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) { + int enabled = (BIF_ARG_2 == am_true); + + if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) { + erts_lcnt_set_preserve_info(enabled); + + BIF_RET(am_ok); + } + } +#endif BIF_ERROR(BIF_P, BADARG); } @@ -4703,8 +4746,8 @@ static void os_info_init(void) void erts_bif_info_init(void) { - erts_smp_atomic_init_nob(&available_internal_state, 0); - erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0); + erts_atomic_init_nob(&available_internal_state, 0); + erts_atomic_init_nob(&hipe_test_reschedule_flag, 0); alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1); alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index ff03151619..3b8e70d44a 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -86,25 +86,25 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) erts_make_ref_in_array(port->async_open_port->ref); port->async_open_port->to = BIF_P->common.id; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); if (ERTS_PROC_PENDING_EXIT(BIF_P)) { /* need to exit caller instead */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_LINK); KILL_CATCHES(BIF_P); BIF_P->freason = EXC_EXIT; erts_port_release(port); BIF_RET(am_badarg); } - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(BIF_P); + ERTS_MSGQ_MV_INQ2PRIVQ(BIF_P); BIF_P->msg.save = BIF_P->msg.last; - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCKS_MSG_RECEIVE); res = erts_proc_store_ref(BIF_P, port->async_open_port->ref); } else { res = port->common.id; - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); } erts_add_link(&ERTS_P_LINKS(port), LINK_PID, BIF_P->common.id); @@ -114,7 +114,7 @@ BIF_RETTYPE erts_internal_open_port_2(BIF_ALIST_2) trace_proc(BIF_P, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_LINK, BIF_P, am_link, port->common.id); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); erts_port_release(port); @@ -271,12 +271,10 @@ BIF_RETTYPE erts_internal_port_call_3(BIF_ALIST_3) break; } - state = erts_smp_atomic32_read_acqb(&BIF_P->state); + state = erts_atomic32_read_acqb(&BIF_P->state); if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { -#ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); -#endif ERTS_BIF_EXITED(BIF_P); } @@ -321,12 +319,10 @@ BIF_RETTYPE erts_internal_port_control_3(BIF_ALIST_3) break; } - state = erts_smp_atomic32_read_acqb(&BIF_P->state); + state = erts_atomic32_read_acqb(&BIF_P->state); if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { -#ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) erts_handle_pending_exit(BIF_P, ERTS_PROC_LOCK_MAIN); -#endif ERTS_BIF_EXITED(BIF_P); } @@ -511,39 +507,35 @@ cleanup_old_port_data(erts_aint_t data) ASSERT(is_immed((Eterm) data)); } else { -#ifdef ERTS_SMP ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; size_t size; - ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, (void *) pdhp, &pdhp->later_op, size); -#else - free_port_data_heap((void *) data); -#endif } } void erts_init_port_data(Port *prt) { - erts_smp_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); + erts_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); } void erts_cleanup_port_data(Port *prt) { ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); - cleanup_old_port_data(erts_smp_atomic_xchg_nob(&prt->data, + cleanup_old_port_data(erts_atomic_xchg_nob(&prt->data, (erts_aint_t) NULL)); } Uint erts_port_data_size(Port *prt) { - erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + erts_aint_t data = erts_atomic_read_ddrb(&prt->data); if ((data & 0x3) != 0) { ASSERT(is_immed((Eterm) (UWord) data)); @@ -558,7 +550,7 @@ erts_port_data_size(Port *prt) ErlOffHeap * erts_port_data_offheap(Port *prt) { - erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + erts_aint_t data = erts_atomic_read_ddrb(&prt->data); if ((data & 0x3) != 0) { ASSERT(is_immed((Eterm) (UWord) data)); @@ -603,11 +595,11 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) ASSERT((data & 0x3) == 0); } - data = erts_smp_atomic_xchg_wb(&prt->data, data); + data = erts_atomic_xchg_wb(&prt->data, data); if (data == (erts_aint_t)NULL) { /* Port terminated by racing thread */ - data = erts_smp_atomic_xchg_wb(&prt->data, data); + data = erts_atomic_xchg_wb(&prt->data, data); ASSERT(data != (erts_aint_t)NULL); cleanup_old_port_data(data); BIF_ERROR(BIF_P, BADARG); @@ -630,7 +622,7 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) if (!prt) BIF_ERROR(BIF_P, BADARG); - data = erts_smp_atomic_read_ddrb(&prt->data); + data = erts_atomic_read_ddrb(&prt->data); if (data == (erts_aint_t)NULL) BIF_ERROR(BIF_P, BADARG); /* Port terminated by racing thread */ @@ -925,7 +917,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); port = erts_open_driver(driver, p->common.id, name_buf, &opts, err_typep, err_nump); #ifdef USE_VM_PROBES @@ -942,7 +934,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) if (port && IS_TRACED_FL(port, F_TRACE_PORTS)) trace_port(port, am_getting_linked, p->common.id); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); if (IS_TRACED_FL(p, F_TRACE_SCHED_PROCS)) { trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in); @@ -1084,7 +1076,7 @@ static byte* convert_environment(Process* p, Eterm env) goto done; } - if ((size = erts_native_filename_need(all,encoding)) < 0) { + if ((size = erts_native_filename_need(all, encoding, 1)) < 0) { goto done; } diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index ad124fd979..bc819505e7 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -66,11 +66,7 @@ static void erts_erts_pcre_stack_free(void *ptr) { #define ERTS_PCRE_STACK_MARGIN (10*1024) -#ifdef ERTS_SMP # define ERTS_STACK_LIMIT ((char *) ethr_get_stacklimit()) -#else -# define ERTS_STACK_LIMIT ((char *) erts_scheduler_stack_limit) -#endif static int stack_guard_downwards(void) diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 45159c4392..22942b40c4 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -60,10 +60,8 @@ static struct { /* Protected by code write permission */ int local; BpFunctions f; /* Local functions */ BpFunctions e; /* Export entries */ -#ifdef ERTS_SMP Process* stager; ErtsThrPrgrLaterOp lop; -#endif } finish_bp; static Eterm @@ -71,9 +69,7 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist); static int erts_set_tracing_event_pattern(Eterm event, Binary*, int on); -#ifdef ERTS_SMP static void smp_bp_finisher(void* arg); -#endif static BIF_RETTYPE system_monitor(Process *p, Eterm monitor_pid, Eterm list); @@ -345,7 +341,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) ERTS_TRACER_CLEAR(&meta_tracer); -#ifdef ERTS_SMP if (finish_bp.current >= 0) { ASSERT(matches >= 0); ASSERT(finish_bp.stager == NULL); @@ -355,7 +350,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) erts_suspend(p, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(p, make_small(matches)); } -#endif erts_release_code_write_permission(); @@ -367,7 +361,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } } -#ifdef ERTS_SMP static void smp_bp_finisher(void* null) { if (erts_finish_breakpointing()) { /* Not done */ @@ -380,15 +373,14 @@ static void smp_bp_finisher(void* null) finish_bp.stager = NULL; #endif erts_release_code_write_permission(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (!ERTS_PROC_IS_EXITING(p)) { erts_resume(p, ERTS_PROC_LOCK_STATUS); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); erts_proc_dec_refc(p); } } -#endif /* ERTS_SMP */ void erts_get_default_trace_pattern(int *trace_pattern_is_on, @@ -397,8 +389,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, struct trace_pattern_flags *trace_pattern_flags, ErtsTracer *meta_tracer) { - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() || - erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_has_code_write_permission() || + erts_thr_progress_is_blocking()); if (trace_pattern_is_on) *trace_pattern_is_on = erts_default_trace_pattern_is_on; if (match_spec) @@ -413,8 +405,8 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, int erts_is_default_trace_enabled(void) { - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission() || - erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_has_code_write_permission() || + erts_thr_progress_is_blocking()); return erts_default_trace_pattern_is_on; } @@ -543,9 +535,7 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) int matches = 0; Uint mask = 0; int cpu_ts = 0; -#ifdef ERTS_SMP int system_blocked = 0; -#endif if (! erts_trace_flags(list, &mask, &tracer, &cpu_ts)) { BIF_ERROR(p, BADARG); @@ -620,13 +610,13 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) goto error; if (start_trace(tracee_p, tracer, &tracee_p->common, on, mask)) { - erts_smp_proc_unlock(tracee_p, + erts_proc_unlock(tracee_p, (tracee_p == p ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); goto already_traced; } - erts_smp_proc_unlock(tracee_p, + erts_proc_unlock(tracee_p, (tracee_p == p ? ERTS_PROC_LOCKS_ALL_MINOR : ERTS_PROC_LOCKS_ALL)); @@ -699,11 +689,9 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) mods = 1; } -#ifdef ERTS_SMP - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); system_blocked = 1; -#endif ok = 1; if (procs || mods) { @@ -766,12 +754,10 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) goto error; } -#ifdef ERTS_SMP if (system_blocked) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); } -#endif erts_release_code_write_permission(); ERTS_TRACER_CLEAR(&tracer); @@ -785,12 +771,10 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) ERTS_TRACER_CLEAR(&tracer); -#ifdef ERTS_SMP if (system_blocked) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); } -#endif erts_release_code_write_permission(); BIF_ERROR(p, BADARG); @@ -878,7 +862,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) trace_flags = ERTS_TRACE_FLAGS(tracee); if (tracee != p) - erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN); } else if (is_external_pid(pid_spec) && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) { return am_undefined; @@ -982,12 +966,12 @@ static int function_is_traced(Process *p, if ((ep = export_get(&e)) != NULL) { pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && - *pc != (BeamInstr) em_call_error_handler) { + ! BeamIsOpCode(*pc, op_call_error_handler)) { int r = 0; - ASSERT(*pc == (BeamInstr) em_apply_bif || - *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_apply_bif) || + BeamIsOpCode(*pc, op_i_generic_breakpoint)); if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; @@ -1055,28 +1039,20 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) mfa[1] = tp[2]; mfa[2] = signed_val(tp[3]); -#ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); } -#endif -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); -#endif + erts_mtx_lock(&erts_dirty_bp_ix_mtx); r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); -#endif -#ifdef ERTS_SMP + erts_mtx_unlock(&erts_dirty_bp_ix_mtx); if ( (key == am_call_time) || (key == am_all)) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); } -#endif switch (r) { case FUNC_TRACE_NOEXIST: @@ -1385,14 +1361,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); #endif - ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1402,8 +1378,8 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, * before turning on breakpoint tracing. */ erts_clear_call_trace_bif(ci, 0); - if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); } } } @@ -1526,17 +1502,13 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, finish_bp.install = on; finish_bp.local = flags.breakpoint; -#ifdef ERTS_SMP if (is_blocking) { - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); -#endif + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); while (erts_finish_breakpointing()) { /* Empty loop body */ } -#ifdef ERTS_SMP finish_bp.current = -1; } -#endif if (flags.breakpoint) { matches += finish_bp.f.matched; @@ -1571,11 +1543,6 @@ erts_set_tracing_event_pattern(Eterm event, Binary* match_spec, int on) finish_bp.f.matched = 0; finish_bp.f.matching = NULL; -#ifndef ERTS_SMP - while (erts_finish_breakpointing()) { - /* Empty loop body */ - } -#endif return 1; } @@ -1594,7 +1561,7 @@ consolidate_event_tracing(ErtsTracingEvent te[]) int erts_finish_breakpointing(void) { - ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); + ERTS_LC_ASSERT(erts_has_code_write_permission()); /* * Memory barriers will be issued for all schedulers *before* @@ -1704,7 +1671,7 @@ uninstall_exp_breakpoints(BpFunctions* f) if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W)); ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1723,7 +1690,7 @@ clean_export_entries(BpFunctions* f) if (ep->addressv[code_ix] == ep->beam) { continue; } - if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) { ep->beam[0] = (BeamInstr) 0; ep->beam[1] = (BeamInstr) 0; } @@ -2015,24 +1982,20 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) } void erts_system_monitor_clear(Process *c_p) { -#ifdef ERTS_SMP if (c_p) { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); } -#endif erts_set_system_monitor(NIL); erts_system_monitor_long_gc = 0; erts_system_monitor_long_schedule = 0; erts_system_monitor_large_heap = 0; erts_system_monitor_flags.busy_port = 0; erts_system_monitor_flags.busy_dist_port = 0; -#ifdef ERTS_SMP if (c_p) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } -#endif } @@ -2142,8 +2105,8 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) int busy_port, busy_dist_port; system_blocked = 1; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0)) goto error; @@ -2182,16 +2145,16 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) erts_system_monitor_flags.busy_port = !!busy_port; erts_system_monitor_flags.busy_dist_port = !!busy_dist_port; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); } error: if (system_blocked) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); } BIF_ERROR(p, BADARG); @@ -2200,23 +2163,19 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) /* Begin: Trace for System Profiling */ void erts_system_profile_clear(Process *c_p) { -#ifdef ERTS_SMP if (c_p) { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); } -#endif erts_set_system_profile(NIL); erts_system_profile_flags.scheduler = 0; erts_system_profile_flags.runnable_procs = 0; erts_system_profile_flags.runnable_ports = 0; erts_system_profile_flags.exclusive = 0; -#ifdef ERTS_SMP if (c_p) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } -#endif } static Eterm system_profile_get(Process *p) { @@ -2278,8 +2237,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) int scheduler, runnable_procs, runnable_ports, exclusive; system_blocked = 1; - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); /* Check if valid process, no locks are taken */ @@ -2330,8 +2289,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) erts_system_profile_flags.runnable_procs = !!runnable_procs; erts_system_profile_flags.exclusive = !!exclusive; erts_system_profile_ts_type = ts; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); BIF_RET(prev); @@ -2339,8 +2298,8 @@ BIF_RETTYPE system_profile_2(BIF_ALIST_2) error: if (system_blocked) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); } BIF_ERROR(p, BADARG); @@ -2365,7 +2324,7 @@ typedef struct { Eterm ref; Eterm ref_heap[ERTS_REF_THING_SIZE]; Eterm target; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; } ErtsTraceDeliveredAll; static void @@ -2373,31 +2332,20 @@ reply_trace_delivered_all(void *vtdarp) { ErtsTraceDeliveredAll *tdarp = (ErtsTraceDeliveredAll *) vtdarp; - if (erts_smp_atomic32_dec_read_nob(&tdarp->refc) == 0) { + if (erts_atomic32_dec_read_nob(&tdarp->refc) == 0) { Eterm ref_copy, msg; Process *rp = tdarp->proc; Eterm *hp = NULL; ErlOffHeap *ohp; -#ifdef ERTS_SMP ErlHeapFragment *bp; bp = new_message_buffer(4 + NC_HEAP_SIZE(tdarp->ref)); hp = &bp->mem[0]; ohp = &bp->off_heap; -#else - ErtsProcLocks rp_locks = 0; - ErtsMessage *mp; - mp = erts_alloc_message_heap( - rp, &rp_locks, 4 + NC_HEAP_SIZE(tdarp->ref), &hp, &ohp); -#endif ref_copy = STORE_NC(&hp, ohp, tdarp->ref); msg = TUPLE3(hp, am_trace_delivered, tdarp->target, ref_copy); -#ifdef ERTS_SMP erts_send_sys_msg_proc(rp->common.id, rp->common.id, msg, bp); -#else - erts_queue_message(rp, rp_locks, mp, msg, am_system); -#endif erts_free(ERTS_ALC_T_MISC_AUX_WORK, vtdarp); erts_proc_dec_refc(rp); @@ -2418,7 +2366,7 @@ trace_delivered_1(BIF_ALIST_1) hp = &tdarp->ref_heap[0]; tdarp->ref = STORE_NC(&hp, NULL, ref); tdarp->target = BIF_ARG_1; - erts_smp_atomic32_init_nob(&tdarp->refc, + erts_atomic32_init_nob(&tdarp->refc, (erts_aint32_t) erts_no_schedulers); erts_proc_add_refc(BIF_P, 1); erts_schedule_multi_misc_aux_work(0, diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index fc6fb5f868..19d46537f9 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -77,11 +77,9 @@ init_reference(void) ref_init_value += (Uint64) tv.tv_usec; #ifdef DEBUG max_thr_id = (Uint32) erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS max_thr_id += (Uint32) erts_no_dirty_cpu_schedulers; max_thr_id += (Uint32) erts_no_dirty_io_schedulers; #endif -#endif erts_atomic64_init_nob(&global_reference.count, (erts_aint64_t) ref_init_value); init_magic_ref_tables(); @@ -136,7 +134,7 @@ Eterm erts_make_ref(Process *c_p) Eterm* hp; Uint32 ref[ERTS_REF_NUMBERS]; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); hp = HAlloc(c_p, ERTS_REF_THING_SIZE); @@ -392,7 +390,8 @@ init_magic_ref_tables(void) erts_snprintf(&tblp->name[0], sizeof(tblp->name), "magic_ref_table_0"); hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); - erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table"); + erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); hash_funcs.hash = nsched_mreft_hash; hash_funcs.cmp = nsched_mreft_cmp; @@ -402,7 +401,8 @@ init_magic_ref_tables(void) erts_snprintf(&tblp->name[0], sizeof(tblp->name), "magic_ref_table_%d", i); hash_init(0, &tblp->hash, &tblp->name[0], 1, hash_funcs); - erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table"); + erts_rwmtx_init(&tblp->rwmtx, "magic_ref_table", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } } @@ -437,10 +437,8 @@ init_unique_integer(void) { int bits; unique_data.r.o.val0_max = (Uint64) erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS unique_data.r.o.val0_max += (Uint64) erts_no_dirty_cpu_schedulers; unique_data.r.o.val0_max += (Uint64) erts_no_dirty_io_schedulers; -#endif bits = erts_fit_in_bits_int64(unique_data.r.o.val0_max); unique_data.r.o.left_shift = bits; unique_data.r.o.right_shift = 64 - bits; @@ -801,7 +799,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0) BIF_RETTYPE res; Eterm* hp; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P)); hp = HAlloc(BIF_P, ERTS_REF_THING_SIZE); diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index b036b28dbf..05007e864e 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -291,7 +291,7 @@ typedef union { * atomics are used they might * differ in size. */ - erts_smp_atomic_t smp_atomic_word; + erts_atomic_t smp_atomic_word; erts_atomic_t atomic_word; } ErtsMagicIndirectionWord; @@ -326,7 +326,7 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, int (*destructor)(Binary *)); ERTS_GLB_INLINE Binary *erts_create_magic_indirection(int (*destructor)(Binary *)); -ERTS_GLB_INLINE erts_smp_atomic_t *erts_smp_binary_to_magic_indirection(Binary *bp); +ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp); ERTS_GLB_INLINE erts_atomic_t *erts_binary_to_magic_indirection(Binary *bp); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -519,16 +519,6 @@ erts_create_magic_indirection(int (*destructor)(Binary *)) but word aligned */ } -ERTS_GLB_INLINE erts_smp_atomic_t * -erts_smp_binary_to_magic_indirection(Binary *bp) -{ - ErtsMagicIndirectionWord *mip; - ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); - ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); - mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); - return &mip->smp_atomic_word; -} - ERTS_GLB_INLINE erts_atomic_t * erts_binary_to_magic_indirection(Binary *bp) { @@ -536,7 +526,7 @@ erts_binary_to_magic_indirection(Binary *bp) ASSERT(bp->intern.flags & BIN_FLAG_MAGIC); ASSERT(ERTS_MAGIC_BIN_ATYPE(bp) == ERTS_ALC_T_MINDIRECTION); mip = ERTS_MAGIC_BIN_UNALIGNED_DATA(bp); - return &mip->atomic_word; + return &mip->smp_atomic_word; } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 71c64997c1..3a16913473 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -32,15 +32,6 @@ #include "erl_bits.h" #include "erl_binary.h" -#ifdef MAX -#undef MAX -#endif -#define MAX(x,y) (((x)>(y))?(x):(y)) -#ifdef MIN -#undef MIN -#endif -#define MIN(x,y) (((x)<(y))?(x):(y)) - #if defined(WORDS_BIGENDIAN) # define BIT_ENDIAN_MACHINE 0 #else @@ -64,30 +55,19 @@ static byte get_bit(byte b, size_t a_offs); -#if defined(ERTS_SMP) /* the state resides in the current process' scheduler data */ -#elif defined(ERL_BITS_REENTRANT) -/* reentrant API but with a hidden single global state, for testing only */ -struct erl_bits_state ErlBitsState_; -#else -/* non-reentrant API with a single global state */ -struct erl_bits_state ErlBitsState; -#endif #define byte_buf (ErlBitsState.byte_buf_) #define byte_buf_len (ErlBitsState.byte_buf_len_) -static erts_smp_atomic_t bits_bufs_size; +static erts_atomic_t bits_bufs_size; Uint erts_bits_bufs_size(void) { - return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size); + return (Uint) erts_atomic_read_nob(&bits_bufs_size); } -#if !defined(ERTS_SMP) -static -#endif void erts_bits_init_state(ERL_BITS_PROTO_0) { @@ -97,13 +77,11 @@ erts_bits_init_state(ERL_BITS_PROTO_0) erts_bin_offset = 0; } -#if defined(ERTS_SMP) void erts_bits_destroy_state(ERL_BITS_PROTO_0) { erts_free(ERTS_ALC_T_BITS_BUF, byte_buf); } -#endif void erts_init_bits(void) @@ -113,13 +91,8 @@ erts_init_bits(void) ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) == offsetof(Binary,orig_bytes)); - erts_smp_atomic_init_nob(&bits_bufs_size, 0); -#if defined(ERTS_SMP) + erts_atomic_init_nob(&bits_bufs_size, 0); /* erl_process.c calls erts_bits_init_state() on all state instances */ -#else - ERL_BITS_DECLARE_STATEP; - erts_bits_init_state(ERL_BITS_ARGS_0); -#endif } /***************************************************************** @@ -753,7 +726,7 @@ static void ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need)) { if (byte_buf_len < need) { - erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len); + erts_atomic_add_nob(&bits_bufs_size, need - byte_buf_len); byte_buf_len = need; byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len); } @@ -1321,7 +1294,14 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, goto badarg; } } + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + c_p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + used_size_in_bits = erts_bin_offset + build_size_in_bits; + sb->is_writable = 0; /* Make sure that no one else can write. */ pb->size = NBYTES(used_size_in_bits); pb->flags |= PB_ACTIVE_WRITER; @@ -1395,9 +1375,21 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, goto badarg; } } - used_size_in_bits = erts_bin_offset + build_size_in_bits; - used_size_in_bytes = NBYTES(used_size_in_bits); - bin_size = 2*used_size_in_bytes; + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + c_p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + + used_size_in_bits = erts_bin_offset + build_size_in_bits; + used_size_in_bytes = NBYTES(used_size_in_bits); + + if(used_size_in_bits < (ERTS_UINT_MAX / 2)) { + bin_size = 2 * used_size_in_bytes; + } else { + bin_size = NBYTES(ERTS_UINT_MAX); + } + bin_size = (bin_size < 256) ? 256 : bin_size; /* @@ -1487,6 +1479,12 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit) * Calculate new size in bytes. */ erts_bin_offset = 8*sb->size + sb->bitsize; + + if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) { + p->freason = SYSTEM_LIMIT; + return THE_NON_VALUE; + } + pos_in_bits_after_build = erts_bin_offset + build_size_in_bits; pb->size = (pos_in_bits_after_build+7) >> 3; pb->flags |= PB_ACTIVE_WRITER; diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 5da2b28a89..b9d141d585 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -84,31 +84,14 @@ typedef struct erl_bin_match_struct{ #define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb) -#if defined(ERTS_SMP) -#define ERL_BITS_REENTRANT -#else -/* uncomment to test the reentrant API in the non-SMP runtime system */ -/* #define ERL_BITS_REENTRANT */ -#endif - -#ifdef ERL_BITS_REENTRANT - /* * Reentrant API with the state passed as a parameter. * (Except when the current Process* already is a parameter.) */ -#ifdef ERTS_SMP /* the state resides in the current process' scheduler data */ #define ERL_BITS_DECLARE_STATEP struct erl_bits_state *EBS #define ERL_BITS_RELOAD_STATEP(P) do{EBS = &erts_proc_sched_data((P))->erl_bits_state;}while(0) #define ERL_BITS_DEFINE_STATEP(P) struct erl_bits_state *EBS = &erts_proc_sched_data((P))->erl_bits_state -#else -/* reentrant API but with a hidden single global state, for testing only */ -extern struct erl_bits_state ErlBitsState_; -#define ERL_BITS_DECLARE_STATEP struct erl_bits_state *EBS = &ErlBitsState_ -#define ERL_BITS_RELOAD_STATEP(P) do{}while(0) -#define ERL_BITS_DEFINE_STATEP(P) ERL_BITS_DECLARE_STATEP -#endif #define ErlBitsState (*EBS) #define ERL_BITS_PROTO_0 struct erl_bits_state *EBS @@ -120,26 +103,6 @@ extern struct erl_bits_state ErlBitsState_; #define ERL_BITS_ARGS_2(ARG1,ARG2) EBS, ARG1, ARG2 #define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3) EBS, ARG1, ARG2, ARG3 -#else /* ERL_BITS_REENTRANT */ - -/* - * Non-reentrant API with a single global state. - */ -extern struct erl_bits_state ErlBitsState; -#define ERL_BITS_DECLARE_STATEP /*empty*/ -#define ERL_BITS_RELOAD_STATEP(P) do{}while(0) -#define ERL_BITS_DEFINE_STATEP(P) /*empty*/ - -#define ERL_BITS_PROTO_0 void -#define ERL_BITS_PROTO_1(PARM1) PARM1 -#define ERL_BITS_PROTO_2(PARM1,PARM2) PARM1, PARM2 -#define ERL_BITS_PROTO_3(PARM1,PARM2,PARM3) PARM1, PARM2, PARM3 -#define ERL_BITS_ARGS_0 /*empty*/ -#define ERL_BITS_ARGS_1(ARG1) ARG1 -#define ERL_BITS_ARGS_2(ARG1,ARG2) ARG1, ARG2 -#define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3) ARG1, ARG2, ARG3 - -#endif /* ERL_BITS_REENTRANT */ #define erts_bin_offset (ErlBitsState.erts_bin_offset_) #define erts_current_bin (ErlBitsState.erts_current_bin_) @@ -158,10 +121,8 @@ extern struct erl_bits_state ErlBitsState; } while (0) void erts_init_bits(void); /* Initialization once. */ -#ifdef ERTS_SMP void erts_bits_init_state(ERL_BITS_PROTO_0); void erts_bits_destroy_state(ERL_BITS_PROTO_0); -#endif /* diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 50f33b2014..49f9beb19f 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -60,7 +60,7 @@ static int max_main_threads; static int reader_groups; static ErtsCpuBindData *scheduler2cpu_map; -static erts_smp_rwmtx_t cpuinfo_rwmtx; +static erts_rwmtx_t cpuinfo_rwmtx; typedef enum { ERTS_CPU_BIND_UNDEFINED, @@ -131,13 +131,11 @@ static erts_cpu_groups_map_t *reader_groups_map; #define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) -#ifdef ERTS_SMP static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, int size, ErtsCpuBindOrder bind_order, int mk_seq); static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); -#endif static void reader_groups_callback(int, ErtsSchedulerData *, int, void *); static erts_cpu_groups_map_t *add_cpu_groups(int groups, @@ -434,7 +432,6 @@ processor_order_cmp(const void *vx, const void *vy) return 0; } -#ifdef ERTS_SMP void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) { @@ -444,7 +441,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) int cgcc_ix; /* Unbind from cpu */ - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); if (scheduler2cpu_map[esdp->no].bound_id >= 0 && erts_unbind_from_cpu(cpuinfo) == 0) { esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; @@ -463,7 +460,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) } } ASSERT(no_cpu_groups_callbacks == cgcc_ix); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) cgcc[cgcc_ix].callback(1, @@ -481,7 +478,7 @@ erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(esdp->run_queue)); if (esdp->no <= max_main_threads) erts_thr_set_main_status(1, (int) esdp->no); @@ -490,7 +487,6 @@ erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) (void) ERTS_RUNQ_FLGS_SET(esdp->run_queue, ERTS_RUNQ_FLG_CHK_CPU_BIND); } -#endif void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) @@ -499,8 +495,8 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_map_t *cgm; erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_callback_call_t *cgcc; - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_runq_unlock(esdp->run_queue); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); cpu_id = scheduler2cpu_map[esdp->no].bind_id; if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { res = erts_bind_to_cpu(cpuinfo, cpu_id); @@ -543,7 +539,7 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) } ASSERT(no_cpu_groups_callbacks == cgcc_ix); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) cgcc[cgcc_ix].callback(0, @@ -553,10 +549,9 @@ erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) erts_free(ERTS_ALC_T_TMP, cgcc); - erts_smp_runq_lock(esdp->run_queue); + erts_runq_lock(esdp->run_queue); } -#ifdef ERTS_SMP void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) { @@ -565,7 +560,7 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_callback_call_t *cgcc; - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); cgcc = erts_alloc(ERTS_ALC_T_TMP, (no_cpu_groups_callbacks @@ -581,7 +576,7 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) } ASSERT(no_cpu_groups_callbacks == cgcc_ix); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) cgcc[cgcc_ix].callback(0, @@ -594,7 +589,6 @@ erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) if (esdp->no <= max_main_threads) erts_thr_set_main_status(1, (int) esdp->no); } -#endif static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) @@ -602,7 +596,7 @@ write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) int s_ix = 1; int cpu_ix; - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) { @@ -702,9 +696,9 @@ Eterm erts_bound_schedulers_term(Process *c_p) { ErtsCpuBindOrder order; - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); order = cpu_bind_order; - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); return bound_schedulers_term(order); } @@ -717,7 +711,7 @@ erts_bind_schedulers(Process *c_p, Eterm how) int cpudata_size; ErtsCpuBindOrder old_cpu_bind_order; - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) { if (cpu_bind_order == ERTS_CPU_BIND_NONE @@ -773,7 +767,7 @@ erts_bind_schedulers(Process *c_p, Eterm how) done: - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); if (notify) erts_sched_notify_check_cpu_bind(); @@ -793,9 +787,9 @@ erts_sched_bind_atthrcreate_child(int unbind) { int res = 0; if (unbind) { - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); res = erts_unbind_from_cpu(cpuinfo); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); } return res; } @@ -812,7 +806,7 @@ erts_sched_bind_atfork_prepare(void) ErtsSchedulerData *esdp = erts_get_scheduler_data(); int unbind = esdp != NULL && erts_is_scheduler_bound(esdp); if (unbind) - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); return unbind; } @@ -820,7 +814,7 @@ int erts_sched_bind_atfork_child(int unbind) { if (unbind) { - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); return erts_unbind_from_cpu(cpuinfo); } @@ -831,7 +825,7 @@ void erts_sched_bind_atfork_parent(int unbind) { if (unbind) - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); } Eterm @@ -865,9 +859,9 @@ erts_fake_scheduler_bindings(Process *p, Eterm how) return res; } - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) ERTS_BIF_PREP_RET(res, am_false); @@ -930,12 +924,12 @@ erts_get_schedulers_binds(Process *c_p) Eterm res = make_tuple(hp); *(hp++) = make_arityval(erts_no_schedulers); - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); for (ix = 1; ix <= erts_no_schedulers; ix++) *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 ? make_small(scheduler2cpu_map[ix].bound_id) : AM_unbound); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); return res; } @@ -1346,7 +1340,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term) int cpudata_size = 0; Eterm res; - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); if (term == am_undefined) { if (user_cpudata) @@ -1367,7 +1361,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term) } else if (is_not_list(term)) { error: - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); res = THE_NON_VALUE; goto done; } @@ -1461,7 +1455,7 @@ erts_set_cpu_topology(Process *c_p, Eterm term) write_schedulers_bind_change(cpudata, cpudata_size); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); erts_sched_notify_check_cpu_bind(); done: @@ -1615,7 +1609,7 @@ erts_get_cpu_topology_term(Process *c_p, Eterm which) { Eterm res; int type; - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); if (ERTS_IS_ATOM_STR("used", which)) type = ERTS_GET_USED_CPU_TOPOLOGY; else if (ERTS_IS_ATOM_STR("detected", which)) @@ -1628,7 +1622,7 @@ erts_get_cpu_topology_term(Process *c_p, Eterm which) res = THE_NON_VALUE; else res = get_cpu_topology_term(c_p, type); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); return res; } @@ -1646,9 +1640,9 @@ get_logical_processors(int *conf, int *onln, int *avail) void erts_get_logical_processors(int *conf, int *onln, int *avail) { - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); get_logical_processors(conf, onln, avail); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); } void @@ -1706,8 +1700,9 @@ erts_init_cpu_topology(void) { int ix; - erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info"); - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_rwmtx_init(&cpuinfo_rwmtx, "cpu_info", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, (sizeof(ErtsCpuBindData) @@ -1725,13 +1720,13 @@ erts_init_cpu_topology(void) NULL); if (cpu_bind_order == ERTS_CPU_BIND_NONE) - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); else { erts_cpu_topology_t *cpudata; int cpudata_size; create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); write_schedulers_bind_change(cpudata, cpudata_size); - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); erts_sched_notify_check_cpu_bind(); destroy_tmp_cpu_topology_copy(cpudata); } @@ -1741,7 +1736,7 @@ int erts_update_cpu_info(void) { int changed; - erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + erts_rwmtx_rwlock(&cpuinfo_rwmtx); changed = erts_cpu_info_update(cpuinfo); if (changed) { erts_cpu_topology_t *cpudata; @@ -1774,7 +1769,7 @@ erts_update_cpu_info(void) write_schedulers_bind_change(cpudata, cpudata_size); destroy_tmp_cpu_topology_copy(cpudata); } - erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_rwmtx_rwunlock(&cpuinfo_rwmtx); if (changed) erts_sched_notify_check_cpu_bind(); return changed; @@ -1791,7 +1786,7 @@ reader_groups_callback(int suspending, void *unused) { if (reader_groups && esdp->no <= max_main_threads) - erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1); + erts_rwmtx_set_reader_group(suspending ? 0 : group+1); } static Eterm get_cpu_groups_map(Process *c_p, @@ -1820,9 +1815,9 @@ Eterm erts_get_reader_groups_map(Process *c_p) { Eterm res; - erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + erts_rwmtx_rlock(&cpuinfo_rwmtx); res = get_cpu_groups_map(c_p, reader_groups_map, 1); - erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + erts_rwmtx_runlock(&cpuinfo_rwmtx); return res; } @@ -2202,7 +2197,7 @@ add_cpu_groups(int groups, erts_cpu_groups_callback_list_t *cgcl; erts_cpu_groups_map_t *cgm; - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); if (use_groups > max_main_threads) use_groups = max_main_threads; @@ -2249,7 +2244,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map, { int start, logical, ix; - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); if (esdp->cpu_id < 0) @@ -2277,7 +2272,7 @@ static void update_cpu_groups_maps(void) { erts_cpu_groups_map_t *cgm; - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) make_cpu_groups_map(cgm, 0); diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h index c922214702..88bcad79ab 100644 --- a/erts/emulator/beam/erl_cpu_topology.h +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -60,11 +60,9 @@ int erts_init_scheduler_bind_type_string(char *how); int erts_init_cpu_topology_string(char *topology_str); void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp); -#ifdef ERTS_SMP void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp); void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp); void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp); -#endif int erts_update_cpu_info(void); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 17e0f2aeec..3ba0886464 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -44,7 +44,7 @@ #include "erl_binary.h" -erts_smp_atomic_t erts_ets_misc_mem_size; +erts_atomic_t erts_ets_misc_mem_size; /* ** Utility macros @@ -61,15 +61,9 @@ enum DbIterSafety { ITER_SAFE_LOCKED, /* Safe while table is locked, not between trap calls */ ITER_SAFE /* No need to fixate at all */ }; -#ifdef ERTS_SMP # define ITERATION_SAFETY(Proc,Tab) \ ((IS_TREE_TABLE((Tab)->common.status) || ONLY_WRITER(Proc,Tab)) ? ITER_SAFE \ : (((Tab)->common.status & DB_FINE_LOCKED) ? ITER_UNSAFE : ITER_SAFE_LOCKED)) -#else -# define ITERATION_SAFETY(Proc,Tab) \ - ((IS_TREE_TABLE((Tab)->common.status) || ONLY_WRITER(Proc,Tab)) \ - ? ITER_SAFE : ITER_SAFE_LOCKED) -#endif #define DID_TRAP(P,Ret) (!is_value(Ret) && ((P)->freason == TRAP)) @@ -195,7 +189,7 @@ static void delete_sched_table(Process *c_p, DbTable *tb); static void table_dec_refc(DbTable *tb, erts_aint_t min_val) { - if (erts_smp_refc_dectest(&tb->common.refc, min_val) == 0) + if (erts_refc_dectest(&tb->common.refc, min_val) == 0) schedule_free_dbtable(tb); } @@ -209,21 +203,21 @@ static ERTS_INLINE void make_btid(DbTable *tb) { Binary *btid = erts_create_magic_indirection(db_table_tid_destructor); - erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); - erts_smp_atomic_init_nob(tbref, (erts_aint_t) tb); + erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid); + erts_atomic_init_nob(tbref, (erts_aint_t) tb); tb->common.btid = btid; /* * Table and magic indirection refer eachother, * and table is refered once by being alive... */ - erts_smp_refc_init(&tb->common.refc, 2); + erts_refc_init(&tb->common.refc, 2); erts_refc_inc(&btid->intern.refc, 1); } static ERTS_INLINE DbTable* btid2tab(Binary* btid) { - erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); - return (DbTable *) erts_smp_atomic_read_nob(tbref); + erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid); + return (DbTable *) erts_atomic_read_nob(tbref); } static DbTable * @@ -231,7 +225,7 @@ tid2tab(Eterm tid) { DbTable *tb; Binary *btid; - erts_smp_atomic_t *tbref; + erts_atomic_t *tbref; if (!is_internal_magic_ref(tid)) return NULL; @@ -239,8 +233,8 @@ tid2tab(Eterm tid) if (ERTS_MAGIC_BIN_DESTRUCTOR(btid) != db_table_tid_destructor) return NULL; - tbref = erts_smp_binary_to_magic_indirection(btid); - tb = (DbTable *) erts_smp_atomic_read_nob(tbref); + tbref = erts_binary_to_magic_indirection(btid); + tb = (DbTable *) erts_atomic_read_nob(tbref); ASSERT(!tb || tb->common.btid == btid); @@ -250,11 +244,11 @@ tid2tab(Eterm tid) static ERTS_INLINE int is_table_alive(DbTable *tb) { - erts_smp_atomic_t *tbref; + erts_atomic_t *tbref; DbTable *rtb; - tbref = erts_smp_binary_to_magic_indirection(tb->common.btid); - rtb = (DbTable *) erts_smp_atomic_read_nob(tbref); + tbref = erts_binary_to_magic_indirection(tb->common.btid); + rtb = (DbTable *) erts_atomic_read_nob(tbref); ASSERT(!rtb || rtb == tb); @@ -264,11 +258,7 @@ is_table_alive(DbTable *tb) static ERTS_INLINE int is_table_named(DbTable *tb) { -#ifdef ERTS_SMP return tb->common.type & DB_NAMED_TABLE; -#else - return tb->common.status & DB_NAMED_TABLE; -#endif } @@ -277,8 +267,8 @@ tid_clear(Process *c_p, DbTable *tb) { DbTable *rtb; Binary *btid = tb->common.btid; - erts_smp_atomic_t *tbref = erts_smp_binary_to_magic_indirection(btid); - rtb = (DbTable *) erts_smp_atomic_xchg_nob(tbref, (erts_aint_t) NULL); + erts_atomic_t *tbref = erts_binary_to_magic_indirection(btid); + rtb = (DbTable *) erts_atomic_xchg_nob(tbref, (erts_aint_t) NULL); ASSERT(!rtb || tb == rtb); if (rtb) { table_dec_refc(tb, 1); @@ -297,13 +287,11 @@ make_tid(Process *c_p, DbTable *tb) /* ** The meta hash table of all NAMED ets tables */ -#ifdef ERTS_SMP # define META_NAME_TAB_LOCK_CNT 16 union { - erts_smp_rwmtx_t lck; + erts_rwmtx_t lck; byte _cache_line_alignment[64]; }meta_name_tab_rwlocks[META_NAME_TAB_LOCK_CNT]; -#endif static struct meta_name_tab_entry { union { Eterm name_atom; @@ -319,13 +307,11 @@ static unsigned meta_name_tab_mask; static ERTS_INLINE struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, - erts_smp_rwmtx_t** lockp) + erts_rwmtx_t** lockp) { unsigned bix = atom_val(name) & meta_name_tab_mask; struct meta_name_tab_entry* bucket = &meta_name_tab[bix]; -#ifdef ERTS_SMP *lockp = &meta_name_tab_rwlocks[bix % META_NAME_TAB_LOCK_CNT].lck; -#endif return bucket; } @@ -390,16 +376,14 @@ free_dbtable(void *vtb) { DbTable *tb = (DbTable *) vtb; #ifdef HARDDEBUG - if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { + if (erts_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) { erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", - erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), + erts_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } #endif -#ifdef ERTS_SMP - erts_smp_rwmtx_destroy(&tb->common.rwlock); - erts_smp_mtx_destroy(&tb->common.fixlock); -#endif + erts_rwmtx_destroy(&tb->common.rwlock); + erts_mtx_destroy(&tb->common.fixlock); ASSERT(is_immed(tb->common.heir_data)); if (tb->common.btid) @@ -419,8 +403,8 @@ static void schedule_free_dbtable(DbTable* tb) * Caller is *not* allowed to access the specialized part * (hash or tree) of *tb after this function has returned. */ - ASSERT(erts_smp_refc_read(&tb->common.refc, 0) == 0); - ASSERT(erts_smp_refc_read(&tb->common.fix_count, 0) == 0); + ASSERT(erts_refc_read(&tb->common.refc, 0) == 0); + ASSERT(erts_refc_read(&tb->common.fix_count, 0) == 0); erts_schedule_thr_prgr_later_cleanup_op(free_dbtable, (void *) tb, &tb->release.data, @@ -435,7 +419,7 @@ save_sched_table(Process *c_p, DbTable *tb) ASSERT(esdp); esdp->ets_tables.count++; - erts_smp_refc_inc(&tb->common.refc, 1); + erts_refc_inc(&tb->common.refc, 1); first = esdp->ets_tables.clist; if (!first) { @@ -525,11 +509,11 @@ save_owned_table(Process *c_p, DbTable *tb) { DbTable *first; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); first = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES); - erts_smp_refc_inc(&tb->common.refc, 1); + erts_refc_inc(&tb->common.refc, 1); if (!first) { tb->common.owned.next = tb->common.owned.prev = tb; @@ -541,13 +525,13 @@ save_owned_table(Process *c_p, DbTable *tb) tb->common.owned.prev->common.owned.next = tb; first->common.owned.prev = tb; } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); } static ERTS_INLINE void delete_owned_table(Process *p, DbTable *tb) { - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (tb->common.owned.next == tb) { DbTable* old; ASSERT(tb->common.owned.prev == tb); @@ -570,39 +554,33 @@ delete_owned_table(Process *p, DbTable *tb) if (tb == first) erts_psd_set(p, ERTS_PSD_ETS_OWNED_TABLES, tb->common.owned.next); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); table_dec_refc(tb, 1); } - -static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, - char *rwname, char* fixname) +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock) { -#ifdef ERTS_SMP - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; if (use_frequent_read_lock) - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; if (erts_ets_rwmtx_spin_count >= 0) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; -#endif -#ifdef ERTS_SMP - erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, - rwname, tb->common.the_name); - erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); + erts_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab", + tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); + erts_mtx_init(&tb->common.fixlock, "db_tab_fix", + tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); -#endif } static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { -#ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { if (kind == LCK_WRITE) { - erts_smp_rwmtx_rwlock(&tb->common.rwlock); + erts_rwmtx_rwlock(&tb->common.rwlock); tb->common.is_thread_safe = 1; } else { - erts_smp_rwmtx_rlock(&tb->common.rwlock); + erts_rwmtx_rlock(&tb->common.rwlock); ASSERT(!tb->common.is_thread_safe); } } @@ -611,14 +589,13 @@ static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) switch (kind) { case LCK_WRITE: case LCK_WRITE_REC: - erts_smp_rwmtx_rwlock(&tb->common.rwlock); + erts_rwmtx_rwlock(&tb->common.rwlock); break; default: - erts_smp_rwmtx_rlock(&tb->common.rwlock); + erts_rwmtx_rlock(&tb->common.rwlock); } ASSERT(tb->common.is_thread_safe); } -#endif } static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) @@ -628,16 +605,15 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) * DbTable structure. That is, ONLY the SMP case is allowed * to follow the tb pointer! */ -#ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { if (kind == LCK_WRITE) { ASSERT(tb->common.is_thread_safe); tb->common.is_thread_safe = 0; - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_rwmtx_rwunlock(&tb->common.rwlock); } else { ASSERT(!tb->common.is_thread_safe); - erts_smp_rwmtx_runlock(&tb->common.rwlock); + erts_rwmtx_runlock(&tb->common.rwlock); } } else { @@ -645,13 +621,12 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) switch (kind) { case LCK_WRITE: case LCK_WRITE_REC: - erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_rwmtx_rwunlock(&tb->common.rwlock); break; default: - erts_smp_rwmtx_runlock(&tb->common.rwlock); + erts_rwmtx_runlock(&tb->common.rwlock); } } -#endif } static ERTS_INLINE @@ -662,7 +637,7 @@ DbTable* db_get_table_aux(Process *p, int meta_already_locked) { DbTable *tb; - erts_smp_rwmtx_t *mtl = NULL; + erts_rwmtx_t *mtl = NULL; /* * IMPORTANT: Only scheduler threads are allowed @@ -674,9 +649,9 @@ DbTable* db_get_table_aux(Process *p, if (is_atom(id)) { struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); if (!meta_already_locked) - erts_smp_rwmtx_rlock(mtl); + erts_rwmtx_rlock(mtl); else{ - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) || erts_lc_rwmtx_is_rwlocked(mtl)); mtl = NULL; } @@ -710,7 +685,7 @@ DbTable* db_get_table_aux(Process *p, } } if (mtl) - erts_smp_rwmtx_runlock(mtl); + erts_rwmtx_runlock(mtl); return tb; } @@ -726,12 +701,12 @@ DbTable* db_get_table(Process *p, static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; - erts_smp_rwmtx_t* rwlock; + erts_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); if (!have_lock) - erts_smp_rwmtx_rwlock(rwlock); + erts_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -779,27 +754,25 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) done: if (!have_lock) - erts_smp_rwmtx_rwunlock(rwlock); + erts_rwmtx_rwunlock(rwlock); return ret; } static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; - erts_smp_rwmtx_t* rwlock; + erts_rwmtx_t* rwlock; Eterm name_atom = tb->common.the_name; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); ASSERT(is_table_named(tb)); -#ifdef ERTS_SMP - if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + if (!have_lock && erts_rwmtx_tryrwlock(rwlock) == EBUSY) { db_unlock(tb, LCK_WRITE); - erts_smp_rwmtx_rwlock(rwlock); + erts_rwmtx_rwlock(rwlock); db_lock(tb, LCK_WRITE); } -#endif - ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); if (bucket->pu.tb == NULL) { goto done; @@ -852,7 +825,7 @@ static int remove_named_tab(DbTable *tb, int have_lock) done: if (!have_lock) - erts_smp_rwmtx_rwunlock(rwlock); + erts_rwmtx_rwunlock(rwlock); return ret; } @@ -861,11 +834,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_smp_refc_inc(&tb->common.fix_count, 1); + erts_refc_inc(&tb->common.fix_count, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_smp_refc_dectest(&tb->common.fix_count, 0) == 0) { + if (erts_refc_dectest(&tb->common.fix_count, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -1506,7 +1479,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) DbTable* tb; Eterm ret; Eterm old_name; - erts_smp_rwmtx_t *lck1, *lck2; + erts_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1529,7 +1502,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) if (lck1 == lck2) lck2 = NULL; else if (lck1 > lck2) { - erts_smp_rwmtx_t *tmp = lck1; + erts_rwmtx_t *tmp = lck1; lck1 = lck2; lck2 = tmp; } @@ -1547,9 +1520,9 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) } } - erts_smp_rwmtx_rwlock(lck1); + erts_rwmtx_rwlock(lck1); if (lck2) - erts_smp_rwmtx_rwlock(lck2); + erts_rwmtx_rwlock(lck2); tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); if (!tb) @@ -1569,16 +1542,16 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) tb->common.the_name = BIF_ARG_2; db_unlock(tb, LCK_WRITE); - erts_smp_rwmtx_rwunlock(lck1); + erts_rwmtx_rwunlock(lck1); if (lck2) - erts_smp_rwmtx_rwunlock(lck2); + erts_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: if (tb) db_unlock(tb, LCK_WRITE); - erts_smp_rwmtx_rwunlock(lck1); + erts_rwmtx_rwunlock(lck1); if (lck2) - erts_smp_rwmtx_rwunlock(lck2); + erts_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1599,9 +1572,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) Uint32 status; Sint keypos; int is_named, is_compressed; -#ifdef ERTS_SMP int is_fine_locked, frequent_read; -#endif #ifdef DEBUG int cret; #endif @@ -1617,10 +1588,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) status = DB_SET | DB_PROTECTED; keypos = 1; is_named = 0; -#ifdef ERTS_SMP is_fine_locked = 0; frequent_read = 0; -#endif heir = am_none; heir_data = (UWord) am_undefined; is_compressed = erts_ets_always_compress; @@ -1648,30 +1617,18 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = signed_val(tp[2]); } else if (tp[1] == am_write_concurrency) { -#ifdef ERTS_SMP if (tp[2] == am_true) { is_fine_locked = 1; } else if (tp[2] == am_false) { is_fine_locked = 0; } else break; -#else - if ((tp[2] != am_true) && (tp[2] != am_false)) { - break; - } -#endif } else if (tp[1] == am_read_concurrency) { -#ifdef ERTS_SMP if (tp[2] == am_true) { frequent_read = 1; } else if (tp[2] == am_false) { frequent_read = 0; } else break; -#else - if ((tp[2] != am_true) && (tp[2] != am_false)) { - break; - } -#endif } else if (tp[1] == am_heir && tp[2] == am_none) { @@ -1713,11 +1670,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) } if (IS_HASH_TABLE(status)) { meth = &db_hash; -#ifdef ERTS_SMP if (is_fine_locked && !(status & DB_PRIVATE)) { status |= DB_FINE_LOCKED; } -#endif } else if (IS_TREE_TABLE(status)) { meth = &db_tree; @@ -1726,10 +1681,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } -#ifdef ERTS_SMP if (frequent_read && !(status & DB_PRIVATE)) status |= DB_FREQ_READ; -#endif /* we create table outside any table lock * and take the unusal cost of destroy table if it @@ -1738,28 +1691,25 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) { DbTable init_tb; - erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0); + erts_atomic_init_nob(&init_tb.common.memory_size, 0); tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE, &init_tb, sizeof(DbTable)); - erts_smp_atomic_init_nob(&tb->common.memory_size, - erts_smp_atomic_read_nob(&init_tb.common.memory_size)); + erts_atomic_init_nob(&tb->common.memory_size, + erts_atomic_read_nob(&init_tb.common.memory_size)); } tb->common.meth = meth; tb->common.the_name = BIF_ARG_1; tb->common.status = status; -#ifdef ERTS_SMP tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ -#endif - erts_smp_refc_init(&tb->common.fix_count, 0); - db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), - "db_tab", "db_tab_fix"); + erts_refc_init(&tb->common.fix_count, 0); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ)); tb->common.keypos = keypos; tb->common.owner = BIF_P->common.id; set_heir(BIF_P, tb, heir, heir_data); - erts_smp_atomic_init_nob(&tb->common.nitems, 0); + erts_atomic_init_nob(&tb->common.nitems, 0); tb->common.fixing_procs = NULL; tb->common.compress = is_compressed; @@ -1942,7 +1892,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) * Process 'rp' might be exiting, but our table lock prevents it * from terminating as it cannot complete erts_db_process_exiting(). */ - ASSERT(!(ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state))); + ASSERT(!(ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state))); delete_owned_table(rp, tb); BIF_P->flags |= F_USING_DB; @@ -2017,12 +1967,12 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3) db_unlock(tb,LCK_WRITE); send_ets_transfer_message(BIF_P, to_proc, &to_locks, tb, BIF_ARG_3); - erts_smp_proc_unlock(to_proc, to_locks); + erts_proc_unlock(to_proc, to_locks); UnUseTmpHeap(5,BIF_P); BIF_RET(am_true); badarg: - if (to_proc != NULL && to_proc != BIF_P) erts_smp_proc_unlock(to_proc, to_locks); + if (to_proc != NULL && to_proc != BIF_P) erts_proc_unlock(to_proc, to_locks); if (tb != NULL) db_unlock(tb, LCK_WRITE); BIF_ERROR(BIF_P, BADARG); } @@ -2246,7 +2196,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { BIF_ERROR(BIF_P, BADARG); } - nitems = erts_smp_atomic_read_nob(&tb->common.nitems); + nitems = erts_atomic_read_nob(&tb->common.nitems); tb->common.meth->db_delete_all_objects(BIF_P, tb); db_unlock(tb, LCK_WRITE); BIF_RET(erts_make_integer(nitems,BIF_P)); @@ -2297,7 +2247,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2) */ struct ErtsEtsAllReq_ { - erts_smp_atomic32_t refc; + erts_atomic32_t refc; Process *proc; ErtsOIRefStorage ref; ErtsEtsAllReqList list[1]; /* one per scheduler */ @@ -2430,7 +2380,7 @@ ets_all_reply(ErtsSchedulerData *esdp, ErtsEtsAllReq **reqpp, erts_proc_dec_refc(reqp->proc); - if (erts_smp_atomic32_dec_read_nob(&reqp->refc) == 0) + if (erts_atomic32_dec_read_nob(&reqp->refc) == 0) erts_free(ERTS_ALC_T_ETS_ALL_REQ, reqp); *reqpp = NULL; @@ -2518,19 +2468,17 @@ BIF_RETTYPE ets_internal_request_all_0(BIF_ALIST_0) Eterm ref = erts_make_ref(BIF_P); ErtsEtsAllReq *req = erts_alloc(ERTS_ALC_T_ETS_ALL_REQ, ERTS_ETS_ALL_REQ_SIZE); - erts_smp_atomic32_init_nob(&req->refc, + erts_atomic32_init_nob(&req->refc, (erts_aint32_t) erts_no_schedulers); erts_oiref_storage_save(&req->ref, ref); req->proc = BIF_P; erts_proc_add_refc(BIF_P, (Sint) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, handle_ets_all_request, (void *) req); -#endif handle_ets_all_request((void *) req); BIF_RET(ref); @@ -3214,7 +3162,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL || tb->common.owner != owner) { if (BIF_P != rp) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) { BIF_RET(am_undefined); } @@ -3228,7 +3176,7 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1) db_unlock(tb, LCK_READ); /*if (rp != NULL && rp != BIF_P) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);*/ + erts_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);*/ hp = HAlloc(BIF_P, 5*sizeof(fields)/sizeof(Eterm)); res = NIL; @@ -3347,11 +3295,10 @@ void init_db(ErtsDbSpinCount db_spin_count) unsigned bits; size_t size; -#ifdef ERTS_SMP int max_spin_count = (1 << 15) - 1; /* internal limit */ - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; switch (db_spin_count) { case ERTS_DB_SPNCNT_NONE: @@ -3391,12 +3338,12 @@ void init_db(ErtsDbSpinCount db_spin_count) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, - "meta_name_tab", make_small(i)); + erts_rwmtx_init_opt(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DB); } -#endif - erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0); + erts_atomic_init_nob(&erts_ets_misc_mem_size, 0); db_initialize_util(); if (user_requested_db_max_tabs < DB_DEF_MAX_TABS) @@ -3497,14 +3444,14 @@ retry: if (tb->common.owner != p->common.id) { if (to_proc != NULL ) { - erts_smp_proc_unlock(to_proc, to_locks); + erts_proc_unlock(to_proc, to_locks); } db_unlock(tb,LCK_WRITE); return !0; /* ok, someone already gave my table away */ } if (tb->common.heir != to_pid) { /* someone changed the heir */ if (to_proc != NULL ) { - erts_smp_proc_unlock(to_proc, to_locks); + erts_proc_unlock(to_proc, to_locks); } if (to_pid == p->common.id || to_pid == am_none) { return 0; /* no real heir, table still mine */ @@ -3517,7 +3464,7 @@ retry: } if (to_proc->common.u.alive.started_interval != tb->common.heir_started_interval) { - erts_smp_proc_unlock(to_proc, to_locks); + erts_proc_unlock(to_proc, to_locks); return 0; /* heir dead and pid reused, table still mine */ } @@ -3534,7 +3481,7 @@ retry: heir_data = tpv[1]; } send_ets_transfer_message(p, to_proc, &to_locks, tb, heir_data); - erts_smp_proc_unlock(to_proc, to_locks); + erts_proc_unlock(to_proc, to_locks); return !0; } @@ -3585,21 +3532,17 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) db_lock(tb, LCK_WRITE_REC); if (!(tb->common.status & DB_DELETE)) { erts_aint_t diff; - #ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); - #endif + erts_mtx_lock(&tb->common.fixlock); ASSERT(fixing_procs_rbt_lookup(tb->common.fixing_procs, p)); diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&tb->common.fix_count,diff,0); + erts_refc_add(&tb->common.fix_count,diff,0); fix->counter = 0; fixing_procs_rbt_delete(&tb->common.fixing_procs, fix); - #ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); - #endif + erts_mtx_unlock(&tb->common.fixlock); if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)) { work += db_unfix_table_hash(&(tb->hash)); } @@ -3607,14 +3550,8 @@ static SWord proc_cleanup_fixed_table(Process* p, DbFixation* fix) ASSERT(sizeof(DbFixation) == ERTS_ALC_DBG_BLK_SZ(fix)); ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof(DbFixation), 0); } - else { - ASSERT(fix->counter == 0); - } db_unlock(tb, LCK_WRITE_REC); } - else { - ASSERT(fix->counter == 0); - } erts_bin_release(fix->tabs.btid); erts_free(ERTS_ALC_T_DB_FIXATION, fix); @@ -3662,9 +3599,9 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) switch (state->op) { case GET_OWNED_TABLE: { DbTable* tb; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); tb = (DbTable*) erts_psd_get(c_p, ERTS_PSD_ETS_OWNED_TABLES); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); if (!tb) { /* Done with owned tables; now fixations */ @@ -3755,10 +3692,8 @@ static void fix_table_locked(Process* p, DbTable* tb) { DbFixation *fix; -#ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); -#endif - erts_smp_refc_inc(&tb->common.fix_count,1); + erts_mtx_lock(&tb->common.fixlock); + erts_refc_inc(&tb->common.fix_count,1); fix = tb->common.fixing_procs; if (fix == NULL) { tb->common.time.monotonic @@ -3771,9 +3706,7 @@ static void fix_table_locked(Process* p, DbTable* tb) ASSERT(fixed_tabs_find(NULL, fix)); ++(fix->counter); -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); -#endif + erts_mtx_unlock(&tb->common.fixlock); return; } } @@ -3786,9 +3719,7 @@ static void fix_table_locked(Process* p, DbTable* tb) fix->counter = 1; fixing_procs_rbt_insert(&tb->common.fixing_procs, fix); -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); -#endif + erts_mtx_unlock(&tb->common.fixlock); p->flags |= F_USING_DB; fixed_tabs_insert(p, fix); @@ -3801,20 +3732,16 @@ static void unfix_table_locked(Process* p, DbTable* tb, { DbFixation* fix; -#ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); -#endif + erts_mtx_lock(&tb->common.fixlock); fix = fixing_procs_rbt_lookup(tb->common.fixing_procs, p); if (fix) { - erts_smp_refc_dec(&tb->common.fix_count,0); + erts_refc_dec(&tb->common.fix_count,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter == 0) { fixing_procs_rbt_delete(&tb->common.fixing_procs, fix); -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); -#endif + erts_mtx_unlock(&tb->common.fixlock); fixed_tabs_delete(p, fix); erts_refc_dec(&fix->tabs.btid->intern.refc, 1); @@ -3825,22 +3752,18 @@ static void unfix_table_locked(Process* p, DbTable* tb, goto unlocked; } } -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); -#endif + erts_mtx_unlock(&tb->common.fixlock); unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) { -#ifdef ERTS_SMP + && erts_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) { if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ - erts_smp_rwmtx_runlock(&tb->common.rwlock); - erts_smp_rwmtx_rwlock(&tb->common.rwlock); + erts_rwmtx_runlock(&tb->common.rwlock); + erts_rwmtx_rwlock(&tb->common.rwlock); *kind_p = LCK_WRITE; if (tb->common.status & DB_DELETE) return; } -#endif db_unfix_table_hash(&(tb->hash)); } } @@ -3856,18 +3779,14 @@ static void free_fixations_op(DbFixation* fix, void* vctx) { struct free_fixations_ctx* ctx = (struct free_fixations_ctx*) vctx; erts_aint_t diff; -#ifdef DEBUG - DbTable* dbg_tb = btid2tab(fix->tabs.btid); -#endif - ASSERT(!dbg_tb || dbg_tb == ctx->tb); + ASSERT(!btid2tab(fix->tabs.btid)); ASSERT(fix->counter > 0); ASSERT(ctx->tb->common.status & DB_DELETE); diff = -((erts_aint_t) fix->counter); - erts_smp_refc_add(&ctx->tb->common.fix_count, diff, 0); + erts_refc_add(&ctx->tb->common.fix_count, diff, 0); -#ifdef ERTS_SMP if (fix->procs.p != ctx->p) { /* Fixated by other process */ fix->counter = 0; @@ -3883,7 +3802,6 @@ static void free_fixations_op(DbFixation* fix, void* vctx) */ } else -#endif { fixed_tabs_delete(fix->procs.p, fix); @@ -3896,7 +3814,6 @@ static void free_fixations_op(DbFixation* fix, void* vctx) ctx->cnt++; } -#ifdef ERTS_SMP int erts_db_execute_free_fixation(Process* p, DbFixation* fix) { ASSERT(fix->counter == 0); @@ -3908,13 +3825,12 @@ int erts_db_execute_free_fixation(Process* p, DbFixation* fix) ERTS_ETS_MISC_MEM_ADD(-sizeof(DbFixation)); return 1; } -#endif static SWord free_fixations_locked(Process* p, DbTable *tb) { struct free_fixations_ctx ctx; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); ctx.p = p; ctx.tb = tb; @@ -4057,7 +3973,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) int use_monotonic; if (What == am_size) { - ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems)); + ret = make_small(erts_atomic_read_nob(&tb->common.nitems)); } else if (What == am_type) { if (tb->common.status & DB_SET) { ret = am_set; @@ -4070,7 +3986,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = am_bag; } } else if (What == am_memory) { - Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + Uint words = (Uint) ((erts_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint)); @@ -4116,9 +4032,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) = ERTS_IS_ATOM_STR("safe_fixed_monotonic_time", What)) || ERTS_IS_ATOM_STR("safe_fixed", What)) { -#ifdef ERTS_SMP - erts_smp_mtx_lock(&tb->common.fixlock); -#endif + erts_mtx_lock(&tb->common.fixlock); if (IS_FIXED(tb)) { Uint need; Eterm *hp; @@ -4160,9 +4074,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) } else { ret = am_false; } -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&tb->common.fixlock); -#endif + erts_mtx_unlock(&tb->common.fixlock); } else if (What == am_atom_put("stats",5)) { if (IS_HASH_TABLE(tb->common.status)) { FloatDef f; @@ -4186,7 +4098,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) std_dev_exp = make_float(hp); PUT_DOUBLE(f, hp); hp += FLOAT_SIZE_OBJECT; - ret = TUPLE7(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)), + ret = TUPLE7(hp, make_small(erts_atomic_read_nob(&tb->hash.nactive)), avg, std_dev_real, std_dev_exp, make_small(stats.min_chain_len), make_small(stats.max_chain_len), @@ -4218,9 +4130,9 @@ static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) tb->common.meth->db_print(to, to_arg, show, tb); - erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems)); + erts_print(to, to_arg, "Objects: %d\n", (int)erts_atomic_read_nob(&tb->common.nitems)); erts_print(to, to_arg, "Words: %bpu\n", - (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size) + (Uint) ((erts_atomic_read_nob(&tb->common.memory_size) + sizeof(Uint) - 1) / sizeof(Uint))); @@ -4260,9 +4172,9 @@ void db_info(fmtfn_t to, void *to_arg, int show) /* Called by break handler * Uint erts_get_ets_misc_mem_size(void) { - ERTS_SMP_MEMORY_BARRIER; + ERTS_THR_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ - return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); + return (Uint) erts_atomic_read_nob(&erts_ets_misc_mem_size); } /* SMP Note: May only be used when system is locked */ @@ -4271,7 +4183,7 @@ erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg) { int ix; - ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(erts_thr_progress_is_blocking()); for (ix = 0; ix < erts_no_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); @@ -4334,3 +4246,47 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } +#ifdef ERTS_ENABLE_LOCK_COUNT + +void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable) { + if(enable) { + erts_lcnt_install_new_lock_info(&tb->common.rwlock.lcnt, "db_tab", + tb->common.the_name, ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB); + erts_lcnt_install_new_lock_info(&tb->common.fixlock.lcnt, "db_tab_fix", + tb->common.the_name, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB); + } else { + erts_lcnt_uninstall(&tb->common.rwlock.lcnt); + erts_lcnt_uninstall(&tb->common.fixlock.lcnt); + } + + if(IS_HASH_TABLE(tb->common.status)) { + erts_lcnt_enable_db_hash_lock_count(&tb->hash, enable); + } +} + +static void lcnt_update_db_locks_per_sched(void *enable) { + ErtsSchedulerData *esdp; + DbTable *head; + + esdp = erts_get_scheduler_data(); + head = esdp->ets_tables.clist; + + if(head) { + DbTable *iterator = head; + + do { + if(is_table_alive(iterator)) { + erts_lcnt_enable_db_lock_count(iterator, !!enable); + } + + iterator = iterator->common.all.next; + } while (iterator != head); + } +} + +void erts_lcnt_update_db_locks(int enable) { + erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, + &lcnt_update_db_locks_per_sched, (void*)(UWord)enable); +} + +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 4ff9f224e8..318e90cb28 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -124,11 +124,16 @@ extern Export ets_select_delete_continue_exp; extern Export ets_select_count_continue_exp; extern Export ets_select_replace_continue_exp; extern Export ets_select_continue_exp; -extern erts_smp_atomic_t erts_ets_misc_mem_size; +extern erts_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); Uint erts_db_get_max_tabs(void); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable); +void erts_lcnt_update_db_locks(int enable); +#endif + #endif /* ERL_DB_H__ */ #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) @@ -146,11 +151,11 @@ do { \ erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ - erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__); \ + erts_atomic_add_nob(&(TAB)->common.memory_size, sz__); \ } while (0) #define ERTS_ETS_MISC_MEM_ADD(SZ) \ - erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ)); + erts_atomic_add_nob(&erts_ets_misc_mem_size, (SZ)); ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type, DbTable *tab, @@ -287,7 +292,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size) ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0); ASSERT(((void *) tab) != ptr - || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0); + || erts_atomic_read_nob(&tab->common.memory_size) == 0); erts_free(type, ptr); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 0addfaa3c7..5d49b2ea14 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -109,22 +109,18 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#ifdef ERTS_SMP # define DB_USING_FINE_LOCKING(TB) (((TB))->common.type & DB_FINE_LOCKED) -#else -# define DB_USING_FINE_LOCKING(TB) 0 -#endif #ifdef ETHR_ORDERED_READ_DEPEND -#define SEGTAB(tb) ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab)) +#define SEGTAB(tb) ((struct segment**) erts_atomic_read_nob(&(tb)->segtab)) #else #define SEGTAB(tb) \ (DB_USING_FINE_LOCKING(tb) \ - ? ((struct segment**) erts_smp_atomic_read_ddrb(&(tb)->segtab)) \ - : ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab))) + ? ((struct segment**) erts_atomic_read_ddrb(&(tb)->segtab)) \ + : ((struct segment**) erts_atomic_read_nob(&(tb)->segtab))) #endif -#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) -#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) +#define NACTIVE(tb) ((int)erts_atomic_read_nob(&(tb)->nactive)) +#define NITEMS(tb) ((int)erts_atomic_read_nob(&(tb)->common.nitems)) #define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) @@ -142,12 +138,12 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) { Uint mask = (DB_USING_FINE_LOCKING(tb) - ? erts_smp_atomic_read_acqb(&tb->szm) - : erts_smp_atomic_read_nob(&tb->szm)); + ? erts_atomic_read_acqb(&tb->szm) + : erts_atomic_read_nob(&tb->szm)); Uint ix = hval & mask; - if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) { + if (ix >= erts_atomic_read_nob(&tb->nactive)) { ix &= mask>>1; - ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive)); + ASSERT(ix < erts_atomic_read_nob(&tb->nactive)); } return ix; } @@ -166,7 +162,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion)); fixd->slot = ix; - was_next = erts_smp_atomic_read_acqb(&tb->fixdel); + was_next = erts_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic insertion in linked list: */ if (NFIXED(tb) <= fixated_by_me) { erts_db_free(ERTS_ALC_T_DB_FIX_DEL, (DbTable*)tb, @@ -175,7 +171,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, } exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg_mb(&tb->fixdel, + was_next = erts_atomic_cmpxchg_mb(&tb->fixdel, (erts_aint_t) fixd, exp_next); }while (was_next != exp_next); @@ -191,62 +187,55 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ make_internal_hash(term, 0)) % MAX_HASH) -#ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) # define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) # define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ -static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) +static ERTS_INLINE erts_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) { if (tb->common.is_thread_safe) { return NULL; } else { - erts_smp_rwmtx_t* lck = GET_LOCK(tb,hval); + erts_rwmtx_t* lck = GET_LOCK(tb,hval); ASSERT(tb->common.type & DB_FINE_LOCKED); - erts_smp_rwmtx_rlock(lck); + erts_rwmtx_rlock(lck); return lck; } } /* Fine grained write lock */ -static ERTS_INLINE erts_smp_rwmtx_t* WLOCK_HASH(DbTableHash* tb, HashValue hval) +static ERTS_INLINE erts_rwmtx_t* WLOCK_HASH(DbTableHash* tb, HashValue hval) { if (tb->common.is_thread_safe) { return NULL; } else { - erts_smp_rwmtx_t* lck = GET_LOCK(tb,hval); + erts_rwmtx_t* lck = GET_LOCK(tb,hval); ASSERT(tb->common.type & DB_FINE_LOCKED); - erts_smp_rwmtx_rwlock(lck); + erts_rwmtx_rwlock(lck); return lck; } } -static ERTS_INLINE void RUNLOCK_HASH(erts_smp_rwmtx_t* lck) +static ERTS_INLINE void RUNLOCK_HASH(erts_rwmtx_t* lck) { if (lck != NULL) { - erts_smp_rwmtx_runlock(lck); + erts_rwmtx_runlock(lck); } } -static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck) +static ERTS_INLINE void WUNLOCK_HASH(erts_rwmtx_t* lck) { if (lck != NULL) { - erts_smp_rwmtx_rwunlock(lck); + erts_rwmtx_rwunlock(lck); } } -#else /* ERTS_SMP */ -# define RLOCK_HASH(tb,hval) NULL -# define WLOCK_HASH(tb,hval) NULL -# define RUNLOCK_HASH(lck) ((void)lck) -# define WUNLOCK_HASH(lck) ((void)lck) -#endif /* ERTS_SMP */ #ifdef ERTS_ENABLE_LOCK_CHECK # define IFN_EXCL(tb,cmd) (((tb)->common.is_thread_safe) || (cmd)) -# define IS_HASH_RLOCKED(tb,hval) IFN_EXCL(tb,erts_smp_lc_rwmtx_is_rlocked(GET_LOCK(tb,hval))) -# define IS_HASH_WLOCKED(tb,lck) IFN_EXCL(tb,erts_smp_lc_rwmtx_is_rwlocked(lck)) -# define IS_TAB_WLOCKED(tb) erts_smp_lc_rwmtx_is_rwlocked(&(tb)->common.rwlock) +# define IS_HASH_RLOCKED(tb,hval) IFN_EXCL(tb,erts_lc_rwmtx_is_rlocked(GET_LOCK(tb,hval))) +# define IS_HASH_WLOCKED(tb,lck) IFN_EXCL(tb,erts_lc_rwmtx_is_rwlocked(lck)) +# define IS_TAB_WLOCKED(tb) erts_lc_rwmtx_is_rwlocked(&(tb)->common.rwlock) #else # define IS_HASH_RLOCKED(tb,hval) (1) # define IS_HASH_WLOCKED(tb,hval) (1) @@ -259,38 +248,27 @@ static ERTS_INLINE void WUNLOCK_HASH(erts_smp_rwmtx_t* lck) ** Slot READ locks updated accordingly, unlocked if EOT. */ static ERTS_INLINE Sint next_slot(DbTableHash* tb, Uint ix, - erts_smp_rwmtx_t** lck_ptr) + erts_rwmtx_t** lck_ptr) { -#ifdef ERTS_SMP ix += DB_HASH_LOCK_CNT; if (ix < NACTIVE(tb)) return ix; RUNLOCK_HASH(*lck_ptr); ix = (ix + 1) & DB_HASH_LOCK_MASK; if (ix != 0) *lck_ptr = RLOCK_HASH(tb,ix); return ix; -#else - return (++ix < NACTIVE(tb)) ? ix : 0; -#endif } /* Same as next_slot but with WRITE locking */ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, - erts_smp_rwmtx_t** lck_ptr) + erts_rwmtx_t** lck_ptr) { -#ifdef ERTS_SMP ix += DB_HASH_LOCK_CNT; if (ix < NACTIVE(tb)) return ix; WUNLOCK_HASH(*lck_ptr); ix = (ix + 1) & DB_HASH_LOCK_MASK; if (ix != 0) *lck_ptr = WLOCK_HASH(tb,ix); return ix; -#else - return next_slot(tb,ix,lck_ptr); -#endif } -#ifndef MIN -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#endif /* * Some special binary flags @@ -334,9 +312,7 @@ struct segment { /* An extended segment table */ struct ext_segtab { -#ifdef ERTS_SMP ErtsThrPrgrLaterOp lop; -#endif struct segment** prev_segtab; /* Used when table is shrinking */ int prev_nsegs; /* Size of prev_segtab */ int nsegs; /* Size of this segtab */ @@ -350,9 +326,9 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, struct segment** segtab) { if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); + erts_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); else - erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); + erts_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); } /* Used by select_replace on analyze_pattern */ @@ -364,7 +340,7 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); -static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, +static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); @@ -562,7 +538,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + if (erts_atomic_cmpxchg_relb(&tb->fixdel, (erts_aint_t) fixdel, (erts_aint_t) NULL) != (erts_aint_t) NULL) { /* Oboy, must join lists */ @@ -571,13 +547,13 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) erts_aint_t exp_tail; while (last->next != NULL) last = last->next; - was_tail = erts_smp_atomic_read_acqb(&tb->fixdel); + was_tail = erts_atomic_read_acqb(&tb->fixdel); do { /* Lockless atomic list insertion */ exp_tail = was_tail; last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel, + was_tail = erts_atomic_cmpxchg_relb(&tb->fixdel, (erts_aint_t) fixdel, exp_tail); }while (was_tail != exp_tail); @@ -593,18 +569,18 @@ SWord db_unfix_table_hash(DbTableHash *tb) FixedDeletion* fixdel; SWord work = 0; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rwlocked(&tb->common.rwlock) - || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock) + || (erts_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg_mb(&tb->fixdel, + fixdel = (FixedDeletion*) erts_atomic_xchg_mb(&tb->fixdel, (erts_aint_t) NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; HashDbTerm **bp; HashDbTerm *b; - erts_smp_rwmtx_t* lck = WLOCK_HASH(tb,ix); + erts_rwmtx_t* lck = WLOCK_HASH(tb,ix); if (IS_FIXED(tb)) { /* interrupted by fixer */ WUNLOCK_HASH(lck); @@ -650,10 +626,10 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); - erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); - erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); - erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); + erts_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); + erts_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); + erts_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); + erts_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; tb->nslots = FIRST_SEGSZ; @@ -662,32 +638,30 @@ int db_create_hash(Process *p, DbTable *tbl) SIZEOF_SEGMENT(FIRST_SEGSZ)); sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ)); -#ifdef ERTS_SMP - erts_smp_atomic_init_nob(&tb->is_resizing, 0); + erts_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; int i; if (tb->common.type & DB_FREQ_READ) - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; if (erts_ets_rwmtx_spin_count >= 0) rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, - "db_hash_slot", tb->common.the_name); + erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB); } /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: */ - ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); + ASSERT(erts_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); } else { /* coarse locking */ tb->locks = NULL; } ERTS_THR_MEMORY_BARRIER; -#endif /* ERST_SMP */ return DB_ERROR_NONE; } @@ -695,7 +669,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret) { DbTableHash *tb = &tbl->hash; Uint ix = 0; - erts_smp_rwmtx_t* lck = RLOCK_HASH(tb,ix); + erts_rwmtx_t* lck = RLOCK_HASH(tb,ix); HashDbTerm* list; for (;;) { @@ -728,7 +702,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) HashValue hval; Uint ix; HashDbTerm* b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); @@ -775,7 +749,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) HashDbTerm** bp; HashDbTerm* b; HashDbTerm* q; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int nitems; int ret = DB_ERROR_NONE; @@ -801,7 +775,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) if (tb->common.status & DB_SET) { HashDbTerm* bnext = b->next; if (b->hvalue == INVALID_HASH) { - erts_smp_atomic_inc_nob(&tb->common.nitems); + erts_atomic_inc_nob(&tb->common.nitems); } else if (key_clash_fail) { ret = DB_ERROR_BADKEY; @@ -829,7 +803,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) do { if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { - erts_smp_atomic_inc_nob(&tb->common.nitems); + erts_atomic_inc_nob(&tb->common.nitems); q->hvalue = hval; if (q != b) { /* must move to preserve key insertion order */ *qp = q->next; @@ -850,7 +824,7 @@ Lnew: q->hvalue = hval; q->next = b; *bp = q; - nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + nitems = erts_atomic_inc_read_nob(&tb->common.nitems); WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); @@ -894,7 +868,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) HashValue hval; int ix; HashDbTerm* b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); @@ -920,7 +894,7 @@ static int db_member_hash(DbTable *tbl, Eterm key, Eterm *ret) HashValue hval; int ix; HashDbTerm* b1; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; hval = MAKE_HASH(key); ix = hash_to_ix(tb, hval); @@ -949,7 +923,7 @@ static int db_get_element_hash(Process *p, DbTable *tbl, HashValue hval; int ix; HashDbTerm* b1; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int retval; hval = MAKE_HASH(key); @@ -1014,7 +988,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) int ix; HashDbTerm** bp; HashDbTerm* b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int nitems_diff = 0; hval = MAKE_HASH(key); @@ -1046,7 +1020,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + erts_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1063,7 +1037,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) int ix; HashDbTerm** bp; HashDbTerm* b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int nitems_diff = 0; int nkeys = 0; Eterm key; @@ -1104,7 +1078,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + erts_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } *ret = am_true; @@ -1115,7 +1089,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) { DbTableHash *tb = &tbl->hash; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; Sint slot; int retval; int nactive; @@ -1230,18 +1204,13 @@ static int match_traverse(Process* p, DbTableHash* tb, unsigned current_list_pos = 0; /* Prefound buckets list index */ Eterm match_res; Sint got = 0; /* Matched terms counter */ - erts_smp_rwmtx_t* lck; /* Slot lock */ + erts_rwmtx_t* lck; /* Slot lock */ int ret_value; -#ifdef ERTS_SMP - erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) + erts_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); - void (*unlock_hash_function)(erts_smp_rwmtx_t*) + void (*unlock_hash_function)(erts_rwmtx_t*) = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH); -#else - #define lock_hash_function(tb, hval) NULL - #define unlock_hash_function(lck) ((void)lck) -#endif - Sint (*next_slot_function)(DbTableHash*, Uint, erts_smp_rwmtx_t**) + Sint (*next_slot_function)(DbTableHash*, Uint, erts_rwmtx_t**) = (lock_for_write ? next_slot_w : next_slot); if ((ret_value = analyze_pattern(tb, pattern, extra_match_validator, &mpi)) @@ -1359,10 +1328,6 @@ done: } return ret_value; -#ifndef SMP -#undef lock_hash_function -#undef unlock_hash_function -#endif } /* @@ -1389,18 +1354,13 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, */ HashDbTerm* saved_current; /* Helper to avoid double skip on match */ Eterm match_res; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int ret_value; -#ifdef ERTS_SMP - erts_smp_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) + erts_rwmtx_t* (*lock_hash_function)(DbTableHash*, HashValue) = (lock_for_write ? WLOCK_HASH : RLOCK_HASH); - void (*unlock_hash_function)(erts_smp_rwmtx_t*) + void (*unlock_hash_function)(erts_rwmtx_t*) = (lock_for_write ? WUNLOCK_HASH : RUNLOCK_HASH); -#else - #define lock_hash_function(tb, hval) NULL - #define unlock_hash_function(lck) ((void)lck) -#endif - Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_smp_rwmtx_t** lck_ptr) + Sint (*next_slot_function)(DbTableHash* tb, Uint ix, erts_rwmtx_t** lck_ptr) = (lock_for_write ? next_slot_w : next_slot); if (got < 0) { @@ -1475,10 +1435,6 @@ done: */ return ret_value; -#ifndef SMP -#undef lock_hash_function -#undef unlock_hash_function -#endif } @@ -2011,7 +1967,7 @@ static int mtraversal_select_delete_on_match_res(void* context_ptr, Sint slot_ix *current_ptr = (*current_ptr)->next; // replace pointer to term using next free_term(sd_context_ptr->tb, del); } - erts_smp_atomic_dec_nob(&sd_context_ptr->tb->common.nitems); + erts_atomic_dec_nob(&sd_context_ptr->tb->common.nitems); return 1; } @@ -2051,11 +2007,7 @@ static int db_select_delete_hash(Process *p, DbTable *tbl, Eterm tid, Eterm patt sd_context.tid = tid; sd_context.hp = NULL; sd_context.prev_continuation_tptr = NULL; -#ifdef ERTS_SMP sd_context.fixated_by_me = sd_context.tb->common.is_thread_safe ? 0 : 1; /* TODO: something nicer */ -#else - sd_context.fixated_by_me = 0; -#endif sd_context.last_pseudo_delete = (Uint) -1; return match_traverse( @@ -2254,7 +2206,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) DbTableHash *tb = &tbl->hash; HashDbTerm **bp, *b; HashValue hval = MAKE_HASH(key); - erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval); + erts_rwmtx_t *lck = WLOCK_HASH(tb, hval); int ix = hash_to_ix(tb, hval); int nitems_diff = 0; @@ -2283,7 +2235,7 @@ static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) } WUNLOCK_HASH(lck); if (nitems_diff) { - erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + erts_atomic_add_nob(&tb->common.nitems, nitems_diff); try_shrink(tb); } return DB_ERROR_NONE; @@ -2305,7 +2257,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) HashDbTerm* list; int i; - ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb)); + ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb)); for (i = 0; i < NACTIVE(tb); i++) { if ((list = BUCKET(tb,i)) != NULL) { @@ -2316,7 +2268,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) }while(list != NULL); } } - erts_smp_atomic_set_nob(&tb->common.nitems, 0); + erts_atomic_set_nob(&tb->common.nitems, 0); return DB_ERROR_NONE; } @@ -2330,7 +2282,6 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) erts_print(to, to_arg, "Buckets: %d\n", NACTIVE(tb)); -#ifdef ERTS_SMP i = tbl->common.is_thread_safe; /* If crash dumping we set table to thread safe in order to avoid taking any locks */ @@ -2340,9 +2291,6 @@ static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) db_calc_stats_hash(&tbl->hash, &stats); tbl->common.is_thread_safe = i; -#else - db_calc_stats_hash(&tbl->hash, &stats); -#endif erts_print(to, to_arg, "Chain Length Avg: %f\n", stats.avg_chain_len); erts_print(to, to_arg, "Chain Length Max: %d\n", stats.max_chain_len); @@ -2394,8 +2342,8 @@ static int db_free_table_hash(DbTable *tbl) static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) { DbTableHash *tb = &tbl->hash; - FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel); - ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE)); + FixedDeletion* fixdel = (FixedDeletion*) erts_atomic_read_acqb(&tb->fixdel); + ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb) || (tb->common.status & DB_DELETE)); while (fixdel != NULL) { FixedDeletion *fx = fixdel; @@ -2407,11 +2355,11 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (--reds < 0) { - erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel); + erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel); return reds; /* Not done */ } } - erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); + erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); while(tb->nslots != 0) { reds -= EXT_SEGSZ/64 + free_seg(tb, 1); @@ -2423,7 +2371,6 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) return reds; /* Not done */ } } -#ifdef ERTS_SMP if (tb->locks != NULL) { int i; for (i=0; i<DB_HASH_LOCK_CNT; ++i) { @@ -2433,8 +2380,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) (void*)tb->locks, sizeof(DbTableHashFineLocks)); tb->locks = NULL; } -#endif - ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); + ASSERT(erts_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); return reds; /* Done */ } @@ -2533,7 +2479,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, if (!db_has_variable(key)) { /* Bound key */ int ix, search_slot; HashDbTerm** bp; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); ix = hash_to_ix(tb, hval); @@ -2637,14 +2583,12 @@ static void alloc_seg(DbTableHash *tb) tb->nslots += EXT_SEGSZ; } -#ifdef ERTS_SMP static void dealloc_ext_segtab(void* lop_data) { struct ext_segtab* est = (struct ext_segtab*) lop_data; erts_free(ERTS_ALC_T_DB_SEG, est); } -#endif /* Shrink table by freeing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty @@ -2683,7 +2627,6 @@ static int free_seg(DbTableHash *tb, int free_records) SET_SEGTAB(tb, est->prev_segtab); tb->nsegs = est->prev_nsegs; -#ifdef ERTS_SMP if (!tb->common.is_thread_safe) { /* * Table is doing a graceful shrink operation and we must avoid @@ -2701,7 +2644,6 @@ static int free_seg(DbTableHash *tb, int free_records) sz); } else -#endif erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, SIZEOF_EXT_SEGTAB(est->nsegs)); } @@ -2762,22 +2704,18 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { -#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); else ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); -#endif return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { -#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) erts_atomic_set_relb(&tb->is_resizing, 0); -#endif } /* Grow table with one or more new buckets. @@ -2788,7 +2726,7 @@ static void grow(DbTableHash* tb, int nitems) HashDbTerm** pnext; HashDbTerm** to_pnext; HashDbTerm* p; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int nactive; int from_ix, to_ix; int szm; @@ -2810,7 +2748,7 @@ static void grow(DbTableHash* tb, int nitems) } ASSERT(nactive < tb->nslots); - szm = erts_smp_atomic_read_nob(&tb->szm); + szm = erts_atomic_read_nob(&tb->szm); if (nactive <= szm) { from_ix = nactive & (szm >> 1); } else { @@ -2821,7 +2759,7 @@ static void grow(DbTableHash* tb, int nitems) to_ix = nactive; lck = WLOCK_HASH(tb, from_ix); - ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); + ERTS_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); /* Now a final double check (with the from_ix lock held) * that we did not get raced by a table fixer. */ @@ -2829,12 +2767,12 @@ static void grow(DbTableHash* tb, int nitems) WUNLOCK_HASH(lck); goto abort; } - erts_smp_atomic_set_nob(&tb->nactive, ++nactive); + erts_atomic_set_nob(&tb->nactive, ++nactive); if (from_ix == 0) { if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->szm, szm); + erts_atomic_set_relb(&tb->szm, szm); else - erts_smp_atomic_set_nob(&tb->szm, szm); + erts_atomic_set_nob(&tb->szm, szm); } done_resizing(tb); @@ -2882,7 +2820,7 @@ static void shrink(DbTableHash* tb, int nitems) HashDbTerm** src_bp; HashDbTerm** dst_bp; HashDbTerm** bp; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int src_ix, dst_ix, low_szm; int nactive; int loop_limit = 5; @@ -2895,13 +2833,13 @@ static void shrink(DbTableHash* tb, int nitems) goto abort; /* already done (race) */ } src_ix = nactive - 1; - low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; + low_szm = erts_atomic_read_nob(&tb->szm) >> 1; dst_ix = src_ix & low_szm; ASSERT(dst_ix < src_ix); ASSERT(nactive > FIRST_SEGSZ); lck = WLOCK_HASH(tb, dst_ix); - ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); + ERTS_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); /* Double check for racing table fixers */ if (IS_FIXED(tb)) { WUNLOCK_HASH(lck); @@ -2930,9 +2868,9 @@ static void shrink(DbTableHash* tb, int nitems) *src_bp = NULL; nactive = src_ix; - erts_smp_atomic_set_nob(&tb->nactive, nactive); + erts_atomic_set_nob(&tb->nactive, nactive); if (dst_ix == 0) { - erts_smp_atomic_set_relb(&tb->szm, low_szm); + erts_atomic_set_relb(&tb->szm, low_szm); } WUNLOCK_HASH(lck); @@ -2967,12 +2905,12 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, /* It return the next live object in a table, NULL if no more */ /* In-bucket: RLOCKED */ /* Out-bucket: RLOCKED unless NULL */ -static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, +static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, HashDbTerm *list) { int i; - ERTS_SMP_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr)); + ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr)); for (list = list->next; list != NULL; list = list->next) { if (list->hvalue != INVALID_HASH) @@ -3002,7 +2940,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, DbTableHash *tb = &tbl->hash; HashValue hval; HashDbTerm **bp, *b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int flags = 0; ASSERT(tb->common.status & DB_SET); @@ -3058,7 +2996,7 @@ db_lookup_dbterm_hash(Process *p, DbTable *tbl, Eterm key, Eterm obj, q->next = next; q->hvalue = hval; *bp = b = q; - erts_smp_atomic_inc_nob(&tb->common.nitems); + erts_atomic_inc_nob(&tb->common.nitems); } HRelease(p, hend, htop); @@ -3084,10 +3022,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) DbTableHash *tb = &tbl->hash; HashDbTerm **bp = (HashDbTerm **) handle->bp; HashDbTerm *b = *bp; - erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; + erts_rwmtx_t* lck = (erts_rwmtx_t*) handle->lck; HashDbTerm* free_me = NULL; - ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ + ERTS_LC_ASSERT(IS_HASH_WLOCKED(tb, lck)); /* locked by db_lookup_dbterm_hash */ ASSERT((&b->dbterm == handle->dbterm) == !(tb->common.compress && handle->flags & DB_MUST_RESIZE)); @@ -3101,7 +3039,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) } WUNLOCK_HASH(lck); - erts_smp_atomic_dec_nob(&tb->common.nitems); + erts_atomic_dec_nob(&tb->common.nitems); try_shrink(tb); } else { if (handle->flags & DB_MUST_RESIZE) { @@ -3110,7 +3048,7 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) } if (handle->flags & DB_INC_TRY_GROW) { int nactive; - int nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems); + int nitems = erts_atomic_inc_read_nob(&tb->common.nitems); WUNLOCK_HASH(lck); nactive = NACTIVE(tb); @@ -3138,7 +3076,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl) } else { db_free_table_hash(tbl); db_create_hash(p, tbl); - erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0); + erts_atomic_set_nob(&tbl->hash.common.nitems, 0); } return 0; } @@ -3168,7 +3106,7 @@ void db_foreach_offheap_hash(DbTable *tbl, void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) { HashDbTerm* b; - erts_smp_rwmtx_t* lck; + erts_rwmtx_t* lck; int sum = 0; int sq_sum = 0; int kept_items = 0; @@ -3206,3 +3144,23 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void) return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); } +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable) { + int i; + + if(tb->locks == NULL) { + return; + } + + for(i = 0; i < DB_HASH_LOCK_CNT; i++) { + erts_lcnt_ref_t *ref = &tb->locks->lck_vec[i].lck.lcnt; + + if(enable) { + erts_lcnt_install_new_lock_info(ref, "db_hash_slot", tb->common.the_name, + ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DB); + } else { + erts_lcnt_uninstall(ref); + } + } +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index f491c85d95..7d27609825 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -42,8 +42,8 @@ typedef struct hash_db_term { typedef struct db_table_hash_fine_locks { union { - erts_smp_rwmtx_t lck; - byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_smp_rwmtx_t))]; + erts_rwmtx_t lck; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_rwmtx_t))]; }lck_vec[DB_HASH_LOCK_CNT]; } DbTableHashFineLocks; @@ -51,10 +51,10 @@ typedef struct db_table_hash { DbTableCommon common; /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ - erts_smp_atomic_t szm; /* current size mask. */ - erts_smp_atomic_t nactive; /* Number of "active" slots */ + erts_atomic_t szm; /* current size mask. */ + erts_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + erts_atomic_t segtab; /* The segment table (struct segment**) */ struct segment* first_segtab[1]; /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ @@ -62,11 +62,9 @@ typedef struct db_table_hash { int nsegs; /* Size of segment table */ /* List of slots where elements have been deleted while table was fixed */ - erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ -#ifdef ERTS_SMP - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ + erts_atomic_t fixdel; /* (FixedDeletion*) */ + erts_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; -#endif } DbTableHash; @@ -103,4 +101,8 @@ typedef struct { void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); Eterm erts_ets_hash_sizeof_ext_segtab(void); +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable); +#endif + #endif /* _DB_HASH_H */ diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index d7deadacf0..5a276b9d88 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -50,7 +50,7 @@ #include "erl_db_tree.h" #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos)) -#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) +#define NITEMS(tb) ((int)erts_atomic_read_nob(&(tb)->common.nitems)) /* ** A stack of this size is enough for an AVL tree with more than @@ -85,16 +85,13 @@ #define EMPTY_NODE(Dtt) (TOP_NODE(Dtt) == NULL) -#ifndef MIN -#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) -#endif /* Obtain table static stack if available. NULL if not. ** Must be released with release_stack() */ static DbTreeStack* get_static_stack(DbTableTree* tb) { - if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { + if (!erts_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } return NULL; @@ -106,7 +103,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb) static DbTreeStack* get_any_stack(DbTableTree* tb) { DbTreeStack* stack; - if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { + if (!erts_atomic_xchg_acqb(&tb->is_stack_busy, 1)) { return &tb->static_stack; } stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -120,8 +117,8 @@ static DbTreeStack* get_any_stack(DbTableTree* tb) static void release_stack(DbTableTree* tb, DbTreeStack* stack) { if (stack == &tb->static_stack) { - ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1); - erts_smp_atomic_set_relb(&tb->is_stack_busy, 0); + ASSERT(erts_atomic_read_nob(&tb->is_stack_busy) == 1); + erts_atomic_set_relb(&tb->is_stack_busy, 0); } else { erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb, @@ -517,7 +514,7 @@ int db_create_tree(Process *p, DbTable *tbl) sizeof(TreeDbTerm *) * STACK_NEED); tb->static_stack.pos = 0; tb->static_stack.slot = 0; - erts_smp_atomic_init_nob(&tb->is_stack_busy, 0); + erts_atomic_init_nob(&tb->is_stack_busy, 0); tb->deletion = 0; return DB_ERROR_NONE; } @@ -646,8 +643,8 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) for (;;) if (!*this) { /* Found our place */ state = 1; - if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { - erts_smp_atomic_dec_nob(&tb->common.nitems); + if (erts_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) { + erts_atomic_dec_nob(&tb->common.nitems); return DB_ERROR_SYSRES; } *this = new_dbterm(tb, obj); @@ -1608,7 +1605,7 @@ static int db_select_delete_continue_tree(Process *p, sc.max = 1000; sc.keypos = tb->common.keypos; - ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy)); + ASSERT(!erts_atomic_read_nob(&tb->is_stack_busy)); traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -2020,7 +2017,7 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds) (DbTable *) tb, (void *) tb->static_stack.array, sizeof(TreeDbTerm *) * STACK_NEED); - ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) + ASSERT(erts_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable)); } return reds; @@ -2030,7 +2027,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl) { db_free_table_tree(tbl); db_create_tree(p, tbl); - erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0); + erts_atomic_set_nob(&tbl->tree.common.nitems, 0); return 0; } @@ -2110,7 +2107,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key) { tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec_nob(&tb->common.nitems); + erts_atomic_dec_nob(&tb->common.nitems); break; } } @@ -2177,7 +2174,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; state = delsub(this); } - erts_smp_atomic_dec_nob(&tb->common.nitems); + erts_atomic_dec_nob(&tb->common.nitems); break; } } diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h index 72749ead1e..dc1b93d410 100644 --- a/erts/emulator/beam/erl_db_tree.h +++ b/erts/emulator/beam/erl_db_tree.h @@ -41,7 +41,7 @@ typedef struct db_table_tree { /* Tree-specific fields */ TreeDbTerm *root; /* The tree root */ Uint deletion; /* Being deleted */ - erts_smp_atomic_t is_stack_busy; + erts_atomic_t is_stack_busy; DbTreeStack static_stack; } DbTableTree; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 13eacaa8a9..e017b9552b 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -170,7 +170,7 @@ static Eterm set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer, Uint d_flags, Uint e_flags) { - ERTS_SMP_LC_ASSERT( + ERTS_LC_ASSERT( ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p) || erts_thr_progress_is_blocking()); @@ -361,11 +361,7 @@ typedef struct { } ErtsMatchPseudoProcess; -#ifdef ERTS_SMP -static erts_smp_tsd_key_t match_pseudo_process_key; -#else -static ErtsMatchPseudoProcess *match_pseudo_process; -#endif +static erts_tsd_key_t match_pseudo_process_key; static ERTS_INLINE void cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) @@ -414,32 +410,27 @@ static ERTS_INLINE ErtsMatchPseudoProcess * get_match_pseudo_process(Process *c_p, Uint heap_size) { ErtsMatchPseudoProcess *mpsp; -#ifdef ERTS_SMP ErtsSchedulerData *esdp; esdp = c_p ? c_p->scheduler_data : erts_get_scheduler_data(); mpsp = esdp ? esdp->match_pseudo_process : - (ErtsMatchPseudoProcess*) erts_smp_tsd_get(match_pseudo_process_key); + (ErtsMatchPseudoProcess*) erts_tsd_get(match_pseudo_process_key); if (mpsp) { - ASSERT(mpsp == erts_smp_tsd_get(match_pseudo_process_key)); + ASSERT(mpsp == erts_tsd_get(match_pseudo_process_key)); ASSERT(mpsp->process.scheduler_data == esdp); cleanup_match_pseudo_process(mpsp, 0); } else { - ASSERT(erts_smp_tsd_get(match_pseudo_process_key) == NULL); + ASSERT(erts_tsd_get(match_pseudo_process_key) == NULL); mpsp = create_match_pseudo_process(); if (esdp) { esdp->match_pseudo_process = (void *) mpsp; } mpsp->process.scheduler_data = esdp; - erts_smp_tsd_set(match_pseudo_process_key, (void *) mpsp); + erts_tsd_set(match_pseudo_process_key, (void *) mpsp); } -#else - mpsp = match_pseudo_process; - cleanup_match_pseudo_process(mpsp, 0); -#endif if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) { mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size); } @@ -449,31 +440,25 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) return mpsp; } -#ifdef ERTS_SMP static void destroy_match_pseudo_process(void) { ErtsMatchPseudoProcess *mpsp; - mpsp = (ErtsMatchPseudoProcess *)erts_smp_tsd_get(match_pseudo_process_key); + mpsp = (ErtsMatchPseudoProcess *)erts_tsd_get(match_pseudo_process_key); if (mpsp) { cleanup_match_pseudo_process(mpsp, 0); erts_free(ERTS_ALC_T_DB_MS_PSDO_PROC, (void *) mpsp); - erts_smp_tsd_set(match_pseudo_process_key, (void *) NULL); + erts_tsd_set(match_pseudo_process_key, (void *) NULL); } } -#endif static void match_pseudo_process_init(void) { -#ifdef ERTS_SMP - erts_smp_tsd_key_create(&match_pseudo_process_key, + erts_tsd_key_create(&match_pseudo_process_key, "erts_match_pseudo_process_key"); - erts_smp_install_exit_handler(destroy_match_pseudo_process); -#else - match_pseudo_process = create_match_pseudo_process(); -#endif + erts_thr_install_exit_handler(destroy_match_pseudo_process); } void @@ -484,7 +469,7 @@ erts_match_set_release_result(Process* c_p) /* The trace control word. */ -static erts_smp_atomic32_t trace_control_word; +static erts_atomic32_t trace_control_word; /* This needs to be here, before the bif table... */ @@ -923,7 +908,7 @@ static void db_free_tmp_uncompressed(DbTerm* obj); */ BIF_RETTYPE db_get_trace_control_word(Process *p) { - Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word); + Uint32 tcw = (Uint32) erts_atomic32_read_acqb(&trace_control_word); BIF_RET(erts_make_integer((Uint) tcw, p)); } @@ -941,7 +926,7 @@ BIF_RETTYPE db_set_trace_control_word(Process *p, Eterm new) if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word, + old_tcw = (Uint32) erts_atomic32_xchg_relb(&trace_control_word, (erts_aint32_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } @@ -1466,7 +1451,7 @@ void db_initialize_util(void){ sizeof(DMCGuardBif), (int (*)(const void *, const void *)) &cmp_guard_bif); match_pseudo_process_init(); - erts_smp_atomic32_init_nob(&trace_control_word, 0); + erts_atomic32_init_nob(&trace_control_word, 0); } @@ -2528,9 +2513,9 @@ restart: case matchEnableTrace: ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); set_tracee_flags(c_p, ERTS_TRACER(c_p), 0, n); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; @@ -2545,9 +2530,9 @@ restart: /* Always take over the tracer of the current process */ set_tracee_flags(tmpp, ERTS_TRACER(c_p), 0, n); if (tmpp == c_p) - erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); else - erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); esp[-1] = am_true; } } @@ -2555,9 +2540,9 @@ restart: case matchDisableTrace: ASSERT(c_p == self); if ( (n = erts_trace_flag2bit(esp[-1]))) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); set_tracee_flags(c_p, ERTS_TRACER(c_p), n, 0); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); esp[-1] = am_true; } else { esp[-1] = FAIL_TERM; @@ -2572,9 +2557,9 @@ restart: /* Always take over the tracer of the current process */ set_tracee_flags(tmpp, ERTS_TRACER(c_p), n, 0); if (tmpp == c_p) - erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL_MINOR); else - erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); esp[-1] = am_true; } } @@ -2598,14 +2583,14 @@ restart: if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) break; if (*esp == am_true) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } else if (*esp == am_false) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) &= ~F_TRACE_SILENT; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } break; case matchTrace2: @@ -2634,10 +2619,10 @@ restart: ERTS_TRACER_CLEAR(&tracer); break; } - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); (--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer, d_flags, e_flags); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACER_CLEAR(&tracer); } break; @@ -2667,13 +2652,13 @@ restart: if (tmpp == c_p) { (--esp)[-1] = set_match_trace(c_p, FAIL_TERM, tracer, d_flags, e_flags); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } else { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); (--esp)[-1] = set_match_trace(tmpp, FAIL_TERM, tracer, d_flags, e_flags); - erts_smp_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(tmpp, ERTS_PROC_LOCKS_ALL); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } ERTS_TRACER_CLEAR(&tracer); } @@ -3277,7 +3262,7 @@ void db_cleanup_offheap_comp(DbTerm* obj) break; case FUN_SUBTAG: ASSERT(u.pb != &tmp); - if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { erts_erase_fun_entry(u.fun->fe); } break; diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 19055c6110..6b126f35d6 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -220,6 +220,9 @@ typedef struct db_fixation { Process* p; } procs; + /* Number of fixations on table from procs.p + * Protected by table write lock or read lock + fixlock + */ Uint counter; } DbFixation; @@ -237,16 +240,14 @@ typedef struct { */ typedef struct db_table_common { - erts_smp_refc_t refc; /* reference count of table struct */ - erts_smp_refc_t fix_count;/* fixation counter */ + erts_refc_t refc; /* reference count of table struct */ + erts_refc_t fix_count;/* fixation counter */ DbTableList all; DbTableList owned; -#ifdef ERTS_SMP - erts_smp_rwmtx_t rwlock; /* rw lock on table */ - erts_smp_mtx_t fixlock; /* Protects fixing_procs and time */ + erts_rwmtx_t rwlock; /* rw lock on table */ + erts_mtx_t fixlock; /* Protects fixing_procs and time */ int is_thread_safe; /* No fine locking inside table needed */ Uint32 type; /* table type, *read only* after creation */ -#endif Eterm owner; /* Pid of the creator */ Eterm heir; /* Pid of the heir */ UWord heir_data; /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */ @@ -254,8 +255,8 @@ typedef struct db_table_common { Eterm the_name; /* an atom */ Binary *btid; DbTableMethod* meth; /* table methods */ - erts_smp_atomic_t nitems; /* Total number of items in table */ - erts_smp_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ + erts_atomic_t nitems; /* Total number of items in table */ + erts_atomic_t memory_size;/* Total memory size. NOTE: in bytes! */ struct { /* Last fixation time */ ErtsMonotonicTime monotonic; ErtsMonotonicTime offset; @@ -288,7 +289,7 @@ typedef struct db_table_common { (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_smp_refc_read(&(T)->common.fix_count,0)) +#define NFIXED(T) (erts_refc_read(&(T)->common.fix_count,0)) #define IS_FIXED(T) (NFIXED(T) != 0) /* diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab index 69421dcfcc..10c76d2579 100644 --- a/erts/emulator/beam/erl_dirty_bif.tab +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -46,6 +46,11 @@ dirty-cpu erts_debug:dirty_cpu/2 dirty-io erts_debug:dirty_io/2 +# lcnt_control/1 doesn't need to be dirty. +dirty-cpu erts_debug:lcnt_control/2 +dirty-cpu erts_debug:lcnt_collect/0 +dirty-cpu erts_debug:lcnt_clear/0 + # --- TEST of Dirty BIF functionality --- # Functions below will execute on dirty schedulers when emulator has # been configured for testing dirty schedulers. This is used for test diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 0e8ebf0c98..d5379a40d5 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -40,7 +40,6 @@ #include "erl_drv_nif.h" #include <stdlib.h> -#include <sys/types.h> /* ssize_t */ #if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_) #ifndef STATIC_ERLANG_DRIVER @@ -48,24 +47,6 @@ #define ERL_DRIVER_TYPES_ONLY #define WIN32_DYNAMIC_ERL_DRIVER #endif -/* - * This structure can be cast to a WSABUF structure. - */ -typedef struct _SysIOVec { - unsigned long iov_len; - char* iov_base; -} SysIOVec; -#else /* Unix */ -# ifdef HAVE_SYS_UIO_H -# include <sys/types.h> -# include <sys/uio.h> -typedef struct iovec SysIOVec; -# else -typedef struct { - char* iov_base; - size_t iov_len; -} SysIOVec; -# endif #endif #ifndef EXTERN @@ -167,14 +148,6 @@ typedef struct _erl_drv_event* ErlDrvEvent; /* An event to be selected on. */ typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */ typedef struct _erl_drv_port* ErlDrvThreadData; /* Thread data. */ -#if !defined(__WIN32__) && !defined(_WIN32) && !defined(_WIN32_) && !defined(USE_SELECT) -struct erl_drv_event_data { - short events; - short revents; -}; -#endif -typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ - typedef struct { unsigned long megasecs; unsigned long secs; @@ -289,10 +262,7 @@ typedef struct erl_drv_entry { unsigned int *flags); /* Works mostly like 'control', a synchronous call into the driver. */ - void (*event)(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); - /* Called when an event selected by - driver_event() has occurred */ + void (*unused_event_callback)(void); int extended_marker; /* ERL_DRV_EXTENDED_MARKER */ int major_version; /* ERL_DRV_EXTENDED_MAJOR_VERSION */ int minor_version; /* ERL_DRV_EXTENDED_MINOR_VERSION */ @@ -359,8 +329,6 @@ EXTERN void erl_drv_busy_msgq_limits(ErlDrvPort port, ErlDrvSizeT *high); EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on); -EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, - ErlDrvEventData event_data); EXTERN int driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len); EXTERN int driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index f88138063e..31b4817fb1 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -144,8 +144,25 @@ typedef signed int ErlNapiSInt; #define ERTS_NAPI_USEC__ 2 #define ERTS_NAPI_NSEC__ 3 -#endif /* __ERL_DRV_NIF_H__ */ - - - +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +/* + * This structure can be cast to a WSABUF structure. + */ +typedef struct _SysIOVec { + unsigned long iov_len; + char* iov_base; +} SysIOVec; +#else /* Unix */ +# include <sys/types.h> +# ifdef HAVE_SYS_UIO_H +# include <sys/uio.h> +typedef struct iovec SysIOVec; +# else +typedef struct { + char* iov_base; + size_t iov_len; +} SysIOVec; +# endif +#endif +#endif /* __ERL_DRV_NIF_H__ */ diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 0e6aadf568..71d4534ef9 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -50,12 +50,11 @@ fatal_error(int err, char *func) #define ERL_DRV_TSD_EXTRA 10 #define ERL_DRV_INVALID_TSD_KEY INT_MAX -#ifdef USE_THREADS struct ErlDrvMutex_ { ethr_mutex mtx; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif char *name; }; @@ -68,7 +67,7 @@ struct ErlDrvCond_ { struct ErlDrvRWLock_ { ethr_rwmutex rwmtx; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif char *name; }; @@ -85,10 +84,6 @@ struct ErlDrvTid_ { static ethr_tsd_key tid_key; -#else /* USE_THREADS */ -static Uint tsd_len; -static void **tsd; -#endif static ErlDrvTSDKey next_tsd_key; static ErlDrvTSDKey max_used_tsd_key; @@ -97,7 +92,6 @@ static char **used_tsd_keys; static erts_mtx_t tsd_mtx; static char *no_name; -#ifdef USE_THREADS static void thread_exit_handler(void) @@ -122,21 +116,15 @@ erl_drv_thread_wrapper(void *vdtid) return (*dtid->func)(dtid->arg); } -#endif void erl_drv_thr_init(void) { int i; -#ifdef USE_THREADS int res = ethr_tsd_key_create(&tid_key,"erts_tid_key"); if (res == 0) res = ethr_install_exit_handler(thread_exit_handler); if (res != 0) fatal_error(res, "erl_drv_thr_init()"); -#else - tsd_len = 0; - tsd = NULL; -#endif no_name = "unknown"; next_tsd_key = 0; @@ -146,19 +134,19 @@ void erl_drv_thr_init(void) sizeof(char *)*ERL_DRV_TSD_KEYS_INC); for (i = 0; i < ERL_DRV_TSD_KEYS_INC; i++) used_tsd_keys[i] = NULL; - erts_mtx_init(&tsd_mtx, "drv_tsd"); + erts_mtx_init(&tsd_mtx, "drv_tsd", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); } /* * These functions implement the driver thread interface in erl_driver.h. * NOTE: Only use this interface from drivers. From within the emulator use - * either the erl_threads.h, the erl_smp.h or the ethread.h interface. + * either the erl_threads.h or the ethread.h interface. */ ErlDrvMutex * erl_drv_mutex_create(char *name) { -#ifdef USE_THREADS ErlDrvMutex *dmtx = erts_alloc_fnf(ERTS_ALC_T_DRV_MTX, (sizeof(ErlDrvMutex) + (name ? sys_strlen(name) + 1 : 0))); @@ -176,45 +164,36 @@ erl_drv_mutex_create(char *name) dmtx->name = no_name; } #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); + erts_lcnt_init_ref_x(&dmtx->lcnt, dmtx->name, NIL, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); #endif } return dmtx; -#else - return (ErlDrvMutex *) NULL; -#endif } void erl_drv_mutex_destroy(ErlDrvMutex *dmtx) { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&dmtx->lcnt); + erts_lcnt_uninstall(&dmtx->lcnt); #endif res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_mutex_destroy()"); erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); -#endif } char * erl_drv_mutex_name(ErlDrvMutex *dmtx) { -#ifdef USE_THREADS return dmtx ? dmtx->name : NULL; -#else - return NULL; -#endif } int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { -#ifdef USE_THREADS int res; if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_trylock()"); @@ -223,22 +202,17 @@ erl_drv_mutex_trylock(ErlDrvMutex *dmtx) erts_lcnt_trylock(&dmtx->lcnt, res); #endif return res; -#else - return 0; -#endif } void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { -#ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_lock()"); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&dmtx->lcnt); #endif ethr_mutex_lock(&dmtx->mtx); -#endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post(&dmtx->lcnt); #endif @@ -247,20 +221,17 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx) void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { -#ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_unlock()"); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&dmtx->lcnt); #endif ethr_mutex_unlock(&dmtx->mtx); -#endif } ErlDrvCond * erl_drv_cond_create(char *name) { -#ifdef USE_THREADS ErlDrvCond *dcnd = erts_alloc_fnf(ERTS_ALC_T_DRV_CND, (sizeof(ErlDrvCond) + (name ? sys_strlen(name) + 1 : 0))); @@ -279,57 +250,43 @@ erl_drv_cond_create(char *name) } } return dcnd; -#else - return (ErlDrvCond *) NULL; -#endif } void erl_drv_cond_destroy(ErlDrvCond *dcnd) { -#ifdef USE_THREADS int res = dcnd ? ethr_cond_destroy(&dcnd->cnd) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_cond_destroy()"); erts_free(ERTS_ALC_T_DRV_CND, (void *) dcnd); -#endif } char * erl_drv_cond_name(ErlDrvCond *dcnd) { -#ifdef USE_THREADS return dcnd ? dcnd->name : NULL; -#else - return NULL; -#endif } void erl_drv_cond_signal(ErlDrvCond *dcnd) { -#ifdef USE_THREADS if (!dcnd) fatal_error(EINVAL, "erl_drv_cond_signal()"); ethr_cond_signal(&dcnd->cnd); -#endif } void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { -#ifdef USE_THREADS if (!dcnd) fatal_error(EINVAL, "erl_drv_cond_broadcast()"); ethr_cond_broadcast(&dcnd->cnd); -#endif } void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { -#ifdef USE_THREADS if (!dcnd || !dmtx) { fatal_error(EINVAL, "erl_drv_cond_wait()"); } @@ -346,13 +303,11 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) break; } } -#endif } ErlDrvRWLock * erl_drv_rwlock_create(char *name) { -#ifdef USE_THREADS ErlDrvRWLock *drwlck = erts_alloc_fnf(ERTS_ALC_T_DRV_RWLCK, (sizeof(ErlDrvRWLock) + (name ? sys_strlen(name) + 1 : 0))); @@ -368,130 +323,106 @@ erl_drv_rwlock_create(char *name) drwlck->name = no_name; } #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); + erts_lcnt_init_ref_x(&drwlck->lcnt, drwlck->name, NIL, + ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); #endif } return drwlck; -#else - return (ErlDrvRWLock *) NULL; -#endif } void erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&drwlck->lcnt); + erts_lcnt_uninstall(&drwlck->lcnt); #endif res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_rwlock_destroy()"); erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); -#endif } char * erl_drv_rwlock_name(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS return drwlck ? drwlck->name : NULL; -#else - return NULL; -#endif } int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); res = ethr_rwmutex_tryrlock(&drwlck->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ); + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTIONS_READ); #endif return res; -#else - return 0; -#endif } void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_rlock(&drwlck->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post(&drwlck->lcnt); #endif -#endif } void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_runlock(&drwlck->rwmtx); -#endif } int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LOCK_OPTIONS_RDWR); #endif return res; -#else - return 0; -#endif } void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwlock(&drwlck->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post(&drwlck->lcnt); #endif -#endif } void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { -#ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwunlock(&drwlck->rwmtx); -#endif } int @@ -585,20 +516,13 @@ erl_drv_tsd_key_destroy(ErlDrvTSDKey key) } -#ifdef USE_THREADS #define ERL_DRV_TSD__ (dtid->tsd) #define ERL_DRV_TSD_LEN__ (dtid->tsd_len) -#else -#define ERL_DRV_TSD__ (tsd) -#define ERL_DRV_TSD_LEN__ (tsd_len) -#endif void erl_drv_tsd_set(ErlDrvTSDKey key, void *data) { -#ifdef USE_THREADS struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) erl_drv_thread_self(); -#endif if (key < 0 || max_used_tsd_key < key || !used_tsd_keys[key]) fatal_error(EINVAL, "erl_drv_tsd_set()"); @@ -626,15 +550,11 @@ erl_drv_tsd_set(ErlDrvTSDKey key, void *data) void * erl_drv_tsd_get(ErlDrvTSDKey key) { -#ifdef USE_THREADS struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key); -#endif if (key < 0 || max_used_tsd_key < key || !used_tsd_keys[key]) fatal_error(EINVAL, "erl_drv_tsd_get()"); -#ifdef USE_THREADS if (!dtid) return NULL; -#endif if (ERL_DRV_TSD_LEN__ <= key) return NULL; return ERL_DRV_TSD__[key]; @@ -669,7 +589,6 @@ erl_drv_thread_create(char *name, void* arg, ErlDrvThreadOpts *opts) { -#ifdef USE_THREADS int res; struct ErlDrvTid_ *dtid; ethr_thr_opts ethr_opts = ETHR_THR_OPTS_DEFAULT_INITER; @@ -711,27 +630,19 @@ erl_drv_thread_create(char *name, *tid = (ErlDrvTid) dtid; return 0; -#else - return ENOTSUP; -#endif } char * erl_drv_thread_name(ErlDrvTid tid) { -#ifdef USE_THREADS struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid; return dtid ? dtid->name : NULL; -#else - return NULL; -#endif } ErlDrvTid erl_drv_thread_self(void) { -#ifdef USE_THREADS struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key); if (!dtid) { int res; @@ -750,15 +661,11 @@ erl_drv_thread_self(void) fatal_error(res, "erl_drv_thread_self()"); } return (ErlDrvTid) dtid; -#else - return (ErlDrvTid) NULL; -#endif } int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2) { -#ifdef USE_THREADS int res; struct ErlDrvTid_ *dtid1 = (struct ErlDrvTid_ *) tid1; struct ErlDrvTid_ *dtid2 = (struct ErlDrvTid_ *) tid2; @@ -772,28 +679,22 @@ erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2) : !ethr_equal_tids(dtid1->tid, dtid2->tid)); return res; -#else - return 1; -#endif } void erl_drv_thread_exit(void *res) { -#ifdef USE_THREADS struct ErlDrvTid_ *dtid = ethr_tsd_get(tid_key); if (dtid && dtid->drv_thr) { ethr_thr_exit(res); fatal_error(0, "erl_drv_thread_exit()"); } -#endif fatal_error(EACCES, "erl_drv_thread_exit()"); } int erl_drv_thread_join(ErlDrvTid tid, void **respp) { -#ifdef USE_THREADS int res; struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid; @@ -806,12 +707,9 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) if (res == 0) erts_free(ERTS_ALC_T_DRV_TID, dtid); return res; -#else - return ENOTSUP; -#endif } -#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP) +#if defined(__DARWIN__) extern int erts_darwin_main_thread_pipe[2]; extern int erts_darwin_main_thread_result_pipe[2]; diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index d18016c42e..9c866250bb 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -30,17 +30,16 @@ static Hash erts_fun_table; -#include "erl_smp.h" #ifdef HIPE # include "hipe_mode_switch.h" #endif -static erts_smp_rwmtx_t erts_fun_table_lock; +static erts_rwmtx_t erts_fun_table_lock; -#define erts_fun_read_lock() erts_smp_rwmtx_rlock(&erts_fun_table_lock) -#define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) -#define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) -#define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) +#define erts_fun_read_lock() erts_rwmtx_rlock(&erts_fun_table_lock) +#define erts_fun_read_unlock() erts_rwmtx_runlock(&erts_fun_table_lock) +#define erts_fun_write_lock() erts_rwmtx_rwlock(&erts_fun_table_lock) +#define erts_fun_write_unlock() erts_rwmtx_rwunlock(&erts_fun_table_lock) static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -59,11 +58,12 @@ void erts_init_fun_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); + erts_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; @@ -113,9 +113,9 @@ erts_put_fun_entry(Eterm mod, int uniq, int index) fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); sys_memset(fe->uniq, 0, sizeof(fe->uniq)); fe->index = 0; - refc = erts_smp_refc_inctest(&fe->refc, 0); + refc = erts_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ - erts_smp_refc_inc(&fe->refc, 1); + erts_refc_inc(&fe->refc, 1); erts_fun_write_unlock(); return fe; } @@ -137,9 +137,9 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq)); fe->index = index; fe->arity = arity; - refc = erts_smp_refc_inctest(&fe->refc, 0); + refc = erts_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ - erts_smp_refc_inc(&fe->refc, 1); + erts_refc_inc(&fe->refc, 1); erts_fun_write_unlock(); return fe; } @@ -164,9 +164,9 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); if (ret) { - erts_aint_t refc = erts_smp_refc_inctest(&ret->refc, 1); + erts_aint_t refc = erts_refc_inctest(&ret->refc, 1); if (refc < 2) /* Pending delete */ - erts_smp_refc_inc(&ret->refc, 1); + erts_refc_inc(&ret->refc, 1); } erts_fun_read_unlock(); return ret; @@ -182,13 +182,11 @@ void erts_erase_fun_entry(ErlFunEntry* fe) { erts_fun_write_lock(); -#ifdef ERTS_SMP /* * We have to check refc again since someone might have looked up * the fun entry and incremented refc after last check. */ - if (erts_smp_refc_dectest(&fe->refc, -1) <= 0) -#endif + if (erts_refc_dectest(&fe->refc, -1) <= 0) { if (fe->address != unloaded_fun) erts_exit(ERTS_ERROR_EXIT, @@ -220,7 +218,7 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) if (start <= addr && addr < end) { fe->pend_purge_address = addr; - ERTS_SMP_WRITE_MEMORY_BARRIER; + ERTS_THR_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; #ifdef HIPE fe->pend_purge_native_address = fe->native_address; @@ -274,10 +272,10 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) #ifdef HIPE fe->pend_purge_native_address = NULL; #endif - if (erts_smp_refc_dectest(&fe->refc, 0) == 0) + if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } - ERTS_SMP_WRITE_MEMORY_BARRIER; + ERTS_THR_WRITE_MEMORY_BARRIER; } void @@ -306,7 +304,7 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg) #ifdef HIPE erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); #endif - erts_print(to, to_arg, "Refc: %ld\n", erts_smp_refc_read(&fe->refc, 1)); + erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); b = b->next; } } @@ -337,7 +335,7 @@ fun_alloc(ErlFunEntry* template) obj->old_uniq = template->old_uniq; obj->old_index = template->old_index; obj->module = template->module; - erts_smp_refc_init(&obj->refc, -1); + erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; obj->pend_purge_address = NULL; #ifdef HIPE diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 289d0d0b28..fb2901d866 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -21,7 +21,7 @@ #ifndef __ERLFUNTABLE_H__ #define __ERLFUNTABLE_H__ -#include "erl_smp.h" +#include "erl_threads.h" /* * Fun entry. @@ -42,7 +42,7 @@ typedef struct erl_fun_entry { Uint arity; /* The arity of the fun. */ Eterm module; /* Tagged atom for module. */ - erts_smp_refc_t refc; /* Reference count: One for code + one for each + erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ #ifdef HIPE diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 3c8bdaa62e..97a1ca915f 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -180,15 +180,13 @@ typedef struct { Eterm ref; Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; } ErtsGCInfoReq; -#ifdef ERTS_DIRTY_SCHEDULERS static struct { erts_mtx_t mtx; ErtsGCInfo info; } dirty_gc; -#endif static ERTS_INLINE int gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) @@ -273,10 +271,9 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info"); + erts_mtx_init(&dirty_gc.mtx, "dirty_gc_info", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); init_gc_info(&dirty_gc.info); -#endif init_gcireq_alloc(); } @@ -340,7 +337,7 @@ erts_heap_sizes(Process* p) for (i = num_heap_sizes-1; i >= 0; i--) { n += 2; - if (!MY_IS_SSMALL(heap_sizes[i])) { + if (!IS_SSMALL(heap_sizes[i])) { big += BIG_UINT_HEAP_SIZE; } } @@ -355,7 +352,7 @@ erts_heap_sizes(Process* p) Eterm num; Sint sz = heap_sizes[i]; - if (MY_IS_SSMALL(sz)) { + if (IS_SSMALL(sz)) { num = make_small(sz); } else { num = uint_to_big(sz, bigp); @@ -480,12 +477,10 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int } if (need == 0) { -#ifdef ERTS_DIRTY_SCHEDULERS if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))); goto force_reschedule; } -#endif return 1; } /* @@ -540,9 +535,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int p->heap_hfrag = hfrag; #endif -#ifdef ERTS_DIRTY_SCHEDULERS force_reschedule: -#endif /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; @@ -615,7 +608,6 @@ young_gen_usage(Process *p) } \ } while (0) -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE void check_for_possibly_long_gc(Process *p, Uint ygen_usage) @@ -639,7 +631,6 @@ check_for_possibly_long_gc(Process *p, Uint ygen_usage) } } -#endif /* * Garbage collect a process. @@ -671,24 +662,20 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds); - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) { -#ifdef ERTS_DIRTY_SCHEDULERS delay_gc_before_start: -#endif return delay_garbage_collection(p, live_hf_end, need, fcalls); } ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p); -#ifdef ERTS_DIRTY_SCHEDULERS if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { check_for_possibly_long_gc(p, ygen_usage); if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) goto delay_gc_before_start; } -#endif if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -697,7 +684,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC); - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); @@ -730,14 +717,12 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { p->flags |= F_NEED_FULLSWEEP; check_for_possibly_long_gc(p, ygen_usage); if (p->flags & F_DIRTY_MAJOR_GC) goto delay_gc_after_start; } -#endif goto do_major_collection; } if (ERTS_SCHEDULER_IS_DIRTY(esdp)) @@ -778,17 +763,15 @@ do_major_collection: ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; int res; - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); erts_send_exit_signal(p, p->common.id, p, &locks, am_kill, NIL, NULL, 0); - erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); -#ifdef ERTS_DIRTY_SCHEDULERS delay_gc_after_start: -#endif /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so we have to remove it after the signal is sent */ - erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); /* We have to make sure that we have space for need on the heap */ res = delay_garbage_collection(p, live_hf_end, need, fcalls); @@ -796,7 +779,7 @@ do_major_collection: return res; } - erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE); @@ -820,7 +803,6 @@ do_major_collection: monitor_large_heap(p); } -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_mtx_lock(&dirty_gc.mtx); dirty_gc.info.garbage_cols++; @@ -828,7 +810,6 @@ do_major_collection: erts_mtx_unlock(&dirty_gc.mtx); } else -#endif { esdp->gc_info.garbage_cols++; esdp->gc_info.reclaimed += reclaimed_now; @@ -906,7 +887,6 @@ garbage_collect_hibernate(Process* p, int check_long_gc) if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); else if (check_long_gc) { @@ -919,11 +899,10 @@ garbage_collect_hibernate(Process* p, int check_long_gc) } p->flags = flags; } -#endif /* * Preliminaries. */ - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); ASSERT(p->stop == p->hend); /* Stack must be empty. */ @@ -1014,7 +993,7 @@ garbage_collect_hibernate(Process* p, int check_long_gc) p->flags |= F_HIBERNATED; - erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); reds = gc_cost(actual_size, actual_size); return reds; @@ -1109,7 +1088,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, p->flags |= F_NEED_FULLSWEEP; -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) p->flags &= ~F_DIRTY_CLA; else { @@ -1125,7 +1103,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, return 10; } } -#endif reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, p->arg_reg, p->arity, fcalls, @@ -1136,7 +1113,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Set GC state. */ - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); + erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); /* * Just did a major collection (which has discarded the old heap), @@ -1283,7 +1260,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, /* * Restore status. */ - erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0); @@ -2913,7 +2890,7 @@ sweep_off_heap(Process *p, int fullsweep) case FUN_SUBTAG: { ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; - if (erts_smp_refc_dectest(&fe->refc, 0) == 0) { + if (erts_refc_dectest(&fe->refc, 0) == 0) { erts_erase_fun_entry(fe); } break; @@ -3229,7 +3206,6 @@ reply_gc_info(void *vgcirp) reclaimed = esdp->gc_info.reclaimed; garbage_cols = esdp->gc_info.garbage_cols; -#ifdef ERTS_DIRTY_SCHEDULERS /* * Add dirty schedulers info on requesting * schedulers info @@ -3240,7 +3216,6 @@ reply_gc_info(void *vgcirp) garbage_cols += dirty_gc.info.garbage_cols; erts_mtx_unlock(&dirty_gc.mtx); } -#endif sz = 0; hpp = NULL; @@ -3273,11 +3248,11 @@ reply_gc_info(void *vgcirp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); - if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0) + if (erts_atomic32_dec_read_nob(&gcirp->refc) == 0) gcireq_free(vgcirp); } @@ -3329,18 +3304,16 @@ erts_gc_info_request(Process *c_p) gcirp->proc = c_p; gcirp->ref = STORE_NC(&hp, NULL, ref); gcirp->req_sched = esdp->no; - erts_smp_atomic32_init_nob(&gcirp->refc, + erts_atomic32_init_nob(&gcirp->refc, (erts_aint32_t) erts_no_schedulers); erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, reply_gc_info, (void *) gcirp); -#endif reply_gc_info((void *) gcirp); @@ -3627,12 +3600,12 @@ erts_check_off_heap2(Process *p, Eterm *htop) refc = erts_refc_read(&u.pb->val->intern.refc, 1); break; case FUN_SUBTAG: - refc = erts_smp_refc_read(&u.fun->fe->refc, 1); + refc = erts_refc_read(&u.fun->fe->refc, 1); break; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - refc = erts_smp_refc_read(&u.ext->node->refc, 1); + refc = erts_refc_read(&u.ext->node->refc, 1); break; case REF_SUBTAG: ASSERT(is_magic_ref_thing(u.hdr)); diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 99995be464..bda2c9b94d 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -96,21 +96,14 @@ typedef enum { #define ERTS_BIF_TIMER_SHORT_TIME 5000 -#ifdef ERTS_SMP -# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore \ - ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore) -#else -# define ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore -#endif - -/* Bit 0 to 9 contains scheduler id (see mask below) */ -#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 10) -#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 11) -#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 12) -#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 13) -#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) -#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) -#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) +/* Bit 0 to 10 contains scheduler id (see mask below) */ +#define ERTS_TMR_ROFLG_HLT (((Uint32) 1) << 11) +#define ERTS_TMR_ROFLG_BIF_TMR (((Uint32) 1) << 12) +#define ERTS_TMR_ROFLG_PRE_ALC (((Uint32) 1) << 13) +#define ERTS_TMR_ROFLG_REG_NAME (((Uint32) 1) << 14) +#define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 15) +#define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 16) +#define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 17) #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -159,7 +152,7 @@ typedef struct { typedef struct { Uint32 roflgs; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; union { void *arg; erts_atomic_t next; @@ -200,7 +193,7 @@ struct ErtsBifTimer_ { ErtsTWTimer twt; } type; struct { - erts_smp_atomic32_t state; + erts_atomic32_t state; #ifdef ERTS_MAGIC_REF_BIF_TIMERS ErtsMagicBinary *mbin; ErtsHLTimerList proc_list; @@ -269,7 +262,6 @@ typedef struct { erts_atomic_t last; } ErtsHLTCncldTmrQTail; -#ifdef ERTS_SMP typedef struct { /* @@ -301,7 +293,6 @@ typedef struct { } head; } ErtsHLTCncldTmrQ; -#endif /* ERTS_SMP */ typedef struct { ErtsHLTimer *root; @@ -309,9 +300,7 @@ typedef struct { } ErtsYieldingTimeoutState; struct ErtsHLTimerService_ { -#ifdef ERTS_SMP ErtsHLTCncldTmrQ canceled_queue; -#endif ErtsHLTimer *time_tree; #ifndef ERTS_MAGIC_REF_BIF_TIMERS ErtsBifTimer *btm_tree; @@ -720,9 +709,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list, #endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ -#ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); -#endif void erts_hl_timer_init(void) @@ -747,9 +734,7 @@ erts_create_timer_service(void) srv->yield = init_yield; erts_twheel_init_timer(&srv->service_timer); -#ifdef ERTS_SMP init_canceled_queue(&srv->canceled_queue); -#endif return srv; } @@ -791,13 +776,13 @@ get_time_left(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos) static ERTS_INLINE int proc_timeout_common(Process *proc, void *tmr) { - if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&proc->common.timer, + if (tmr == (void *) erts_atomic_cmpxchg_mb(&proc->common.timer, ERTS_PTMR_TIMEDOUT, (erts_aint_t) tmr)) { erts_aint32_t state; - erts_smp_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); - state = erts_smp_atomic32_read_acqb(&proc->state); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); + state = erts_atomic32_read_acqb(&proc->state); + erts_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_RECEIVE); if (!(state & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_EXITING))) erts_schedule_process(proc, state, 0); return 1; @@ -808,7 +793,7 @@ proc_timeout_common(Process *proc, void *tmr) static ERTS_INLINE int port_timeout_common(Port *port, void *tmr) { - if (tmr == (void *) erts_smp_atomic_cmpxchg_mb(&port->common.timer, + if (tmr == (void *) erts_atomic_cmpxchg_mb(&port->common.timer, ERTS_PTMR_TIMEDOUT, (erts_aint_t) tmr)) { erts_port_task_schedule(port->common.id, @@ -821,24 +806,24 @@ port_timeout_common(Port *port, void *tmr) #ifdef ERTS_MAGIC_REF_BIF_TIMERS -static erts_smp_atomic_t * +static erts_atomic_t * mbin_to_btmref__(ErtsMagicBinary *mbin) { - return erts_smp_binary_to_magic_indirection((Binary *) mbin); + return erts_binary_to_magic_indirection((Binary *) mbin); } static ERTS_INLINE void magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr) { - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); + erts_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_atomic_init_nob(aptr, (erts_aint_t) tmr); } static ERTS_INLINE ErtsBifTimer * magic_binary_to_btm(ErtsMagicBinary *mbin) { - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr); + erts_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsBifTimer *tmr = (ErtsBifTimer *) erts_atomic_read_nob(aptr); ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); return tmr; } @@ -884,7 +869,7 @@ init_btm_specifics(ErtsSchedulerData *esdp, btm_rbt_insert(&esdp->timer_service->btm_tree, tmr); #endif - erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE); + erts_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE); return refc; /* refc from magic binary... */ } @@ -917,10 +902,10 @@ timer_pre_dec_refc(ErtsTimer *tmr) { #ifdef ERTS_HLT_DEBUG erts_aint_t refc; - refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + refc = erts_atomic32_dec_read_nob(&tmr->head.refc); ERTS_HLT_ASSERT(refc > 0); #else - erts_smp_atomic32_dec_nob(&tmr->head.refc); + erts_atomic32_dec_nob(&tmr->head.refc); #endif } @@ -969,8 +954,8 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) static ERTS_INLINE void tw_timer_dec_refc(ErtsTWTimer *tmr) { - if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { - ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + if (erts_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); schedule_tw_timer_destroy(tmr); } } @@ -1114,7 +1099,7 @@ create_tw_timer(ErtsSchedulerData *esdp, return NULL; } - erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + erts_atomic32_init_nob(&tmr->head.refc, refc); erts_twheel_set_timer(esdp->timer_wheel, &tmr->u.tw_tmr, @@ -1147,7 +1132,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) * at once... */ - ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); + ERTS_HLT_ASSERT(erts_atomic32_read_nob(&tmr->head.refc) == 0); if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name)); @@ -1179,14 +1164,13 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) static ERTS_INLINE void hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) { - if (erts_smp_atomic32_dec_read_relb(&tmr->head.refc) == 0) { - ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + if (erts_atomic32_dec_read_relb(&tmr->head.refc) == 0) { + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); schedule_hl_timer_destroy(tmr, roflgs); } } static void hlt_service_timeout(void *vesdp); -#ifdef ERTS_SMP static void handle_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq, int use_limit, @@ -1194,12 +1178,11 @@ static void handle_canceled_queue(ErtsSchedulerData *esdp, int *need_thr_progress, ErtsThrPrgrVal *thr_prgr_p, int *need_more_work); -#endif static ERTS_INLINE void check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) { -#if defined(ERTS_SMP) && ERTS_TMR_CHECK_CANCEL_ON_CREATE +#if ERTS_TMR_CHECK_CANCEL_ON_CREATE ErtsHLTCncldTmrQ *cq = &srv->canceled_queue; if (cq->head.first != cq->head.unref_end) handle_canceled_queue(esdp, cq, 1, @@ -1219,14 +1202,14 @@ bif_timer_ref_destructor(Binary *unused) static ERTS_INLINE void btm_clear_magic_binary(ErtsBifTimer *tmr) { - erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); + erts_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); Uint32 roflgs = tmr->type.head.roflgs; #ifdef ERTS_HLT_DEBUG - erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, + erts_aint_t tval = erts_atomic_xchg_nob(aptr, (erts_aint_t) NULL); ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr); #else - erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); + erts_atomic_set_nob(aptr, (erts_aint_t) NULL); #endif if (roflgs & ERTS_TMR_ROFLG_HLT) hl_timer_dec_refc(&tmr->type.hlt, roflgs); @@ -1246,7 +1229,7 @@ bif_timer_timeout(ErtsHLTimerService *srv, ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_TIMED_OUT, ERTS_TMR_STATE_ACTIVE); @@ -1273,14 +1256,15 @@ bif_timer_timeout(ErtsHLTimerService *srv, ERTS_HLT_ASSERT(proc); } if (proc) { + int dec_refc = 0; + ErtsMessage *mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + tmr->btm.bp = NULL; + erts_queue_message(proc, 0, mp, tmr->btm.message, + am_clock_service); + erts_proc_lock(proc, ERTS_PROC_LOCK_BTM); + /* If the process is exiting do not disturb the cleanup... */ if (!ERTS_PROC_IS_EXITING(proc)) { - int dec_refc = 0; - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = tmr->btm.bp; - tmr->btm.bp = NULL; - erts_queue_message(proc, 0, mp, tmr->btm.message, - am_clock_service); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->btm.proc_list.next) { proc_btm_list_delete(&proc->bif_timers, tmr); @@ -1293,10 +1277,10 @@ bif_timer_timeout(ErtsHLTimerService *srv, dec_refc = 1; } #endif - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (dec_refc) - timer_pre_dec_refc((ErtsTimer *) tmr); } + erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); } if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); @@ -1431,7 +1415,7 @@ create_hl_timer(ErtsSchedulerData *esdp, } tmr->head.roflgs = roflgs; - erts_smp_atomic32_init_nob(&tmr->head.refc, refc); + erts_atomic32_init_nob(&tmr->head.refc, refc); if (!srv->next_timeout || tmr->timeout < srv->next_timeout->timeout) { @@ -1663,7 +1647,6 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, } } -#ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq) @@ -1793,7 +1776,7 @@ cq_check_incoming(ErtsSchedulerData *esdp, ErtsHLTCncldTmrQ *cq) cq->head.next.thr_progress_reached = 1; /* Move unreferenced end pointer forward... */ - ERTS_HLT_SMP_MEMBAR_LoadLoad_LoadStore; + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); cq->head.unref_end = cq->head.next.unref_end; @@ -1886,31 +1869,24 @@ erts_handle_canceled_timers(void *vesdp, need_more_work); } -#endif /* ERTS_SMP */ static void queue_canceled_timer(ErtsSchedulerData *esdp, int rsched_id, ErtsTimer *tmr) { -#ifdef ERTS_SMP ErtsHLTCncldTmrQ *cq; cq = &ERTS_SCHEDULER_IX(rsched_id-1)->timer_service->canceled_queue; if (cq_enqueue(cq, tmr, rsched_id - (int) esdp->no)) erts_notify_canceled_timer(esdp, rsched_id); -#else - ERTS_INTERNAL_ERROR("Unexpected enqueue of canceled timer"); -#endif } static void continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) { -#ifdef ERTS_SMP Uint32 sid = (tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK); if (esdp->no != sid) queue_canceled_timer(esdp, sid, tmr); else -#endif cleanup_sched_local_canceled_timer(esdp, tmr); } @@ -1996,7 +1972,7 @@ setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, #else proc_btm_rbt_insert(&proc->bif_timers, tmr); #endif - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM); tmr->type.head.receiver.proc = proc; } } @@ -2017,7 +1993,7 @@ cancel_bif_timer(ErtsBifTimer *tmr) Uint32 roflgs; int res; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); if (state != ERTS_TMR_STATE_ACTIVE) @@ -2039,7 +2015,7 @@ cancel_bif_timer(ErtsBifTimer *tmr) proc = tmr->type.head.receiver.proc; ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + erts_proc_lock(proc, ERTS_PROC_LOCK_BTM); /* * If process is exiting, let it clean up * the btm tree by itself (it may be in @@ -2058,7 +2034,7 @@ cancel_bif_timer(ErtsBifTimer *tmr) res = 1; } #endif - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + erts_proc_unlock(proc, ERTS_PROC_LOCK_BTM); } return res; @@ -2081,7 +2057,7 @@ access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr)); if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); + erts_aint32_t state = erts_atomic32_read_acqb(&tmr->btm.state); if (state == ERTS_TMR_STATE_ACTIVE) return get_time_left(esdp, timeout); return -1; @@ -2175,7 +2151,7 @@ send_async_info(Process *proc, ErtsProcLocks initial_locks, locks &= ~initial_locks; if (locks) - erts_smp_proc_unlock(proc, locks); + erts_proc_unlock(proc, locks); return am_ok; } @@ -2261,7 +2237,7 @@ send_sync_info(Process *proc, ErtsProcLocks initial_locks, locks &= ~initial_locks; if (locks) - erts_smp_proc_unlock(proc, locks); + erts_proc_unlock(proc, locks); return am_ok; } @@ -2375,9 +2351,9 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, * Check if the timer is aimed at current * process... */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); + erts_proc_lock(c_p, ERTS_PROC_LOCK_BTM); tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); if (!tmr) return 0; @@ -2418,7 +2394,7 @@ no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info) erts_queue_message(c_p, locks, mp, msg, am_clock_service); locks &= ~ERTS_PROC_LOCK_MAIN; if (locks) - erts_smp_proc_unlock(c_p, locks); + erts_proc_unlock(c_p, locks); return am_ok; } @@ -2494,7 +2470,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) req->rrefn[1] = rrefn[1]; req->rrefn[2] = rrefn[2]; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); if (ERTS_PROC_PENDING_EXIT(c_p)) ERTS_VBUMP_ALL_REDS(c_p); @@ -2512,10 +2488,10 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) * otherwise, next receive will *not* work * as expected! */ - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); c_p->msg.save = c_p->msg.last; } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); ERTS_BIF_PREP_TRAP1(ret, erts_await_result, c_p, rref); } @@ -2605,7 +2581,7 @@ exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) erts_aint_t state; int is_hlt; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + state = erts_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); @@ -2991,7 +2967,7 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, ERTS_TMR_PROC, (void *) c_p, c_p->common.id, THE_NON_VALUE, NULL, NULL, NULL); - erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); + erts_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); } } @@ -3002,7 +2978,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo) ErtsMonotonicTime tmo, timeout_pos; int short_time, tres; - ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + ERTS_HLT_ASSERT(erts_atomic_read_nob(&c_p->common.timer) == ERTS_PTMR_NONE); tres = parse_timeout_pos(esdp, etmo, &tmo, 0, @@ -3022,7 +2998,7 @@ erts_set_proc_timer_uword(Process *c_p, UWord tmo) { ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); - ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) + ERTS_HLT_ASSERT(erts_atomic_read_nob(&c_p->common.timer) == ERTS_PTMR_NONE); #ifndef ARCH_32 @@ -3045,13 +3021,13 @@ void erts_cancel_proc_timer(Process *c_p) { erts_aint_t tval; - tval = erts_smp_atomic_xchg_acqb(&c_p->common.timer, + tval = erts_atomic_xchg_acqb(&c_p->common.timer, ERTS_PTMR_NONE); c_p->flags &= ~(F_INSLPQUEUE|F_TIMO); if (tval == ERTS_PTMR_NONE) return; if (tval == ERTS_PTMR_TIMEDOUT) { - erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE); + erts_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE); return; } continue_cancel_ptimer(erts_proc_sched_data(c_p), @@ -3066,7 +3042,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) ErtsMonotonicTime timeout_pos; ErtsCreateTimerFunc create_timer; - if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) + if (erts_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) erts_cancel_port_timer(c_prt); check_canceled_queue(esdp, esdp->timer_service); @@ -3079,14 +3055,14 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, (void *) c_prt, c_prt->common.id, THE_NON_VALUE, NULL, NULL, NULL); - erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); + erts_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); } void erts_cancel_port_timer(Port *c_prt) { erts_aint_t tval; - tval = erts_smp_atomic_xchg_acqb(&c_prt->common.timer, + tval = erts_atomic_xchg_acqb(&c_prt->common.timer, ERTS_PTMR_NONE); if (tval == ERTS_PTMR_NONE) return; @@ -3094,7 +3070,7 @@ erts_cancel_port_timer(Port *c_prt) while (!erts_port_task_is_scheduled(&c_prt->timeout_task)) erts_thr_yield(); erts_port_task_abort(&c_prt->timeout_task); - erts_smp_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE); + erts_atomic_set_nob(&c_prt->common.timer, ERTS_PTMR_NONE); return; } continue_cancel_ptimer(erts_get_scheduler_data(), @@ -3108,7 +3084,7 @@ erts_read_port_timer(Port *c_prt) erts_aint_t itmr; ErtsMonotonicTime timeout_pos; - itmr = erts_smp_atomic_read_acqb(&c_prt->common.timer); + itmr = erts_atomic_read_acqb(&c_prt->common.timer); if (itmr == ERTS_PTMR_NONE) return (Sint64) -1; if (itmr == ERTS_PTMR_TIMEDOUT) @@ -3245,7 +3221,7 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) return; #endif - if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { + if (erts_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) ? tmr->type.head.receiver.name @@ -3283,7 +3259,7 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, btmfd.func = func; btmfd.arg = arg; - if (!erts_smp_thr_progress_is_blocking()) + if (!erts_thr_progress_is_blocking()) ERTS_INTERNAL_ERROR("Not blocking thread progress"); for (six = 0; six < erts_no_schedulers; six++) { @@ -3374,7 +3350,7 @@ erts_debug_callback_timer_foreach(void (*tclbk)(void *), dfct.func = func; dfct.arg = arg; - if (!erts_smp_thr_progress_is_blocking()) + if (!erts_thr_progress_is_blocking()) ERTS_INTERNAL_ERROR("Not blocking thread progress"); for (six = 0; six < erts_no_schedulers; six++) { diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index ff31f04cb9..e6f5e8b67d 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -36,16 +36,16 @@ typedef struct ErtsHLTimerService_ ErtsHLTimerService; #define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1)) #define ERTS_PTMR_INIT(P) \ - erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE) + erts_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE) #define ERTS_PTMR_IS_SET(P) \ - (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer)) + (ERTS_PTMR_NONE != erts_atomic_read_nob(&(P)->common.timer)) #define ERTS_PTMR_IS_TIMED_OUT(P) \ - (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer)) + (ERTS_PTMR_TIMEDOUT == erts_atomic_read_nob(&(P)->common.timer)) #define ERTS_PTMR_CLEAR(P) \ do { \ ASSERT(ERTS_PTMR_IS_TIMED_OUT((P))); \ - erts_smp_atomic_set_nob(&(P)->common.timer, \ + erts_atomic_set_nob(&(P)->common.timer, \ ERTS_PTMR_NONE); \ } while (0) @@ -63,13 +63,11 @@ void erts_hl_timer_init(void); void erts_start_timer_callback(ErtsMonotonicTime, void (*)(void *), void *); -#ifdef ERTS_SMP void erts_handle_canceled_timers(void *vesdp, int *need_thr_progress, ErtsThrPrgrVal *thr_prgr_p, int *need_more_work); -#endif Uint erts_bif_timer_memory_size(void); void erts_print_bif_timer_info(fmtfn_t to, void *to_arg); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 6172595552..6cef9bd0e3 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -49,6 +49,7 @@ #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_check_io.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -69,23 +70,17 @@ * The variables below (prefixed with etp_) are for erts/etc/unix/etp-commands * only. Do not remove even though they aren't used elsewhere in the emulator! */ -#ifdef ERTS_SMP const int etp_smp_compiled = 1; -#else -const int etp_smp_compiled = 0; -#endif -#ifdef USE_THREADS const int etp_thread_compiled = 1; -#else -const int etp_thread_compiled = 0; -#endif const char etp_erts_version[] = ERLANG_VERSION; const char etp_otp_release[] = ERLANG_OTP_RELEASE; const char etp_compile_date[] = ERLANG_COMPILE_DATE; const char etp_arch[] = ERLANG_ARCHITECTURE; #ifdef ERTS_ENABLE_KERNEL_POLL +const int erts_use_kernel_poll = 1; const int etp_kernel_poll_support = 1; #else +const int erts_use_kernel_poll = 0; const int etp_kernel_poll_support = 0; #endif #if defined(ARCH_64) @@ -156,17 +151,10 @@ static void erl_init(int ncpu, static erts_atomic_t exiting; -#ifdef ERTS_SMP -erts_smp_atomic32_t erts_writing_erl_crash_dump; +erts_atomic32_t erts_writing_erl_crash_dump; erts_tsd_key_t erts_is_crash_dumping_key; -#else -volatile int erts_writing_erl_crash_dump = 0; -#endif int erts_initialized = 0; -#if defined(USE_THREADS) && !defined(ERTS_SMP) -erts_tid_t erts_main_thread; -#endif int erts_use_sender_punish; @@ -185,7 +173,7 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace * in error codes. */ -erts_smp_atomic32_t erts_max_gen_gcs; +erts_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, am_info or am_warning, am_error is @@ -195,11 +183,9 @@ int erts_compat_rel; static int no_schedulers; static int no_schedulers_online; -#ifdef ERTS_DIRTY_SCHEDULERS static int no_dirty_cpu_schedulers; static int no_dirty_cpu_schedulers_online; static int no_dirty_io_schedulers; -#endif #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -220,16 +206,16 @@ int erts_no_line_info = 0; /* -L: Don't load line information */ */ ErtsModifiedTimings erts_modified_timings[] = { - /* 0 */ {make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS}, - /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS}, - /* 2 */ {make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2}, - /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS}, - /* 4 */ {make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS}, - /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2}, - /* 6 */ {make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS}, - /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3}, - /* 8 */ {make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS}, - /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4} + /* 0 */ {make_small(0), CONTEXT_REDS}, + /* 1 */ {make_small(0), (3*CONTEXT_REDS)/4}, + /* 2 */ {make_small(0), CONTEXT_REDS/2}, + /* 3 */ {make_small(0), (7*CONTEXT_REDS)/8}, + /* 4 */ {make_small(0), CONTEXT_REDS/3}, + /* 5 */ {make_small(0), (10*CONTEXT_REDS)/11}, + /* 6 */ {make_small(1), CONTEXT_REDS/4}, + /* 7 */ {make_small(1), (5*CONTEXT_REDS)/7}, + /* 8 */ {make_small(10), CONTEXT_REDS/5}, + /* 9 */ {make_small(10), (6*CONTEXT_REDS)/7} }; #define ERTS_MODIFIED_TIMING_LEVELS \ @@ -327,12 +313,11 @@ erl_init(int ncpu, erts_init_sys_common_misc(); erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, - no_schedulers_online -#ifdef ERTS_DIRTY_SCHEDULERS - , no_dirty_cpu_schedulers, + no_schedulers_online, + erts_no_poll_threads, + no_dirty_cpu_schedulers, no_dirty_cpu_schedulers_online, no_dirty_io_schedulers -#endif ); erts_late_init_time_sup(); erts_init_cpu_topology(); /* Must be after init_scheduling */ @@ -410,7 +395,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** */ erts_init_empty_process(&parent); - erts_smp_proc_lock(&parent, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(&parent, ERTS_PROC_LOCK_MAIN); hp = HAlloc(&parent, argc*2 + 4); args = NIL; for (i = argc-1; i >= 0; i--) { @@ -425,7 +410,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** so.flags = erts_default_spo_flags|SPO_SYSTEM_PROC; res = erl_create_process(&parent, start_mod, am_start, args, &so); - erts_smp_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(&parent, ERTS_PROC_LOCK_MAIN); erts_cleanup_empty_process(&parent); return res; } @@ -450,7 +435,7 @@ erl_system_process_otp(Eterm parent_pid, char* modname, int off_heap_msgq) if (off_heap_msgq) so.flags |= SPO_OFF_HEAP_MSGQ; res = erl_create_process(parent, start_mod, am_start, NIL, &so); - erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(parent, ERTS_PROC_LOCK_MAIN); return res; } @@ -577,9 +562,19 @@ void erts_usage(void) erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); erts_fprintf(stderr, " valid values are: off_heap | on_heap\n"); + erts_fprintf(stderr, "-IOp number set number of pollsets to be used to poll for I/O,\n"); + erts_fprintf(stderr, " This value has to be equal or smaller than the\n"); + erts_fprintf(stderr, " number of poll threads. If the current platform\n"); + erts_fprintf(stderr, " does not support concurrent update of pollsets\n"); + erts_fprintf(stderr, " this value is ignored.\n"); + erts_fprintf(stderr, "-IOt number set number of threads to be used to poll for I/O\n"); + erts_fprintf(stderr, "-IOPp number set number of pollsets as a percentage of the\n"); + erts_fprintf(stderr, " number of poll threads."); + erts_fprintf(stderr, "-IOPt number set number of threads to be used to poll for I/O\n"); + erts_fprintf(stderr, " as a percentage of the number of schedulers."); + /* erts_fprintf(stderr, "-i module set the boot module (default init)\n"); */ - erts_fprintf(stderr, "-K boolean enable or disable kernel poll\n"); erts_fprintf(stderr, "-n[s|a|d] Control behavior of signals to ports\n"); erts_fprintf(stderr, " Note that this flag is deprecated!\n"); erts_fprintf(stderr, "-M<X> <Y> memory allocator switches,\n"); @@ -624,7 +619,6 @@ void erts_usage(void) ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE, ERTS_DEFAULT_SCHED_STACK_SIZE); -#ifdef ERTS_DIRTY_SCHEDULERS erts_fprintf(stderr, "-sssdcpu size suggested stack size in kilo words for dirty CPU scheduler\n"); erts_fprintf(stderr, " threads, valid range is [%d-%d] (default %d)\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, @@ -635,7 +629,6 @@ void erts_usage(void) ERTS_SCHED_THREAD_MIN_STACK_SIZE, ERTS_SCHED_THREAD_MAX_STACK_SIZE, ERTS_DEFAULT_DIO_SCHED_STACK_SIZE); -#endif erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); @@ -644,7 +637,6 @@ void erts_usage(void) erts_fprintf(stderr, "-SP p1:p2 specify schedulers (p1) and schedulers online (p2)\n"); erts_fprintf(stderr, " as percentages of logical processors configured and logical\n"); erts_fprintf(stderr, " processors available, respectively\n"); -#ifdef ERTS_DIRTY_SCHEDULERS erts_fprintf(stderr, "-SDcpu n1:n2 set number of dirty CPU schedulers (n1), and number of\n"); erts_fprintf(stderr, " dirty CPU schedulers online (n2), valid range for both\n"); erts_fprintf(stderr, " numbers is [1-%d], and n2 must be less than or equal to n1\n", @@ -654,7 +646,6 @@ void erts_usage(void) erts_fprintf(stderr, " and logical processors available, respectively\n"); erts_fprintf(stderr, "-SDio n set number of dirty I/O schedulers, valid range is [0-%d]\n", ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); -#endif erts_fprintf(stderr, "-t size set the maximum number of atoms the emulator can handle\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", MIN_ATOM_TABLE_SIZE, MAX_ATOM_TABLE_SIZE); @@ -682,7 +673,6 @@ void erts_usage(void) erts_exit(1, ""); } -#ifdef USE_THREADS /* * allocators for thread lib */ @@ -724,7 +714,6 @@ static void ethr_ll_free(void *ptr) erts_free(ERTS_ALC_T_ETHR_LL, ptr); } -#endif static int early_init(int *argc, char **argv) /* @@ -742,22 +731,16 @@ early_init(int *argc, char **argv) /* int schdlrs_percentage = 100; int schdlrs_onln_percentage = 100; int max_main_threads; -#ifdef ERTS_DIRTY_SCHEDULERS int dirty_cpu_scheds; int dirty_cpu_scheds_online; int dirty_cpu_scheds_pctg = 100; int dirty_cpu_scheds_onln_pctg = 100; int dirty_io_scheds; -#endif int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; -#if defined(USE_THREADS) && !defined(ERTS_SMP) - erts_main_thread = erts_thr_self(); -#endif - erts_save_emu_args(*argc, argv); erts_sched_compact_load = 1; @@ -781,11 +764,6 @@ early_init(int *argc, char **argv) /* &ncpu, &ncpuonln, &ncpuavail); -#ifndef ERTS_SMP - ncpu = 1; - ncpuonln = 1; - ncpuavail = 1; -#endif ignore_break = 0; replace_intr = 0; @@ -797,18 +775,12 @@ early_init(int *argc, char **argv) /* erts_sys_pre_init(); erts_atomic_init_nob(&exiting, 0); -#ifdef ERTS_SMP erts_thr_progress_pre_init(); -#endif -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); + erts_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key"); -#else - erts_writing_erl_crash_dump = 0; -#endif - erts_smp_atomic32_init_nob(&erts_max_gen_gcs, + erts_atomic32_init_nob(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); @@ -825,11 +797,9 @@ early_init(int *argc, char **argv) /* schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; -#ifdef ERTS_DIRTY_SCHEDULERS dirty_cpu_scheds = no_schedulers; dirty_cpu_scheds_online = no_schedulers_online; dirty_io_scheds = 10; -#endif envbufsz = sizeof(envbuf); @@ -882,6 +852,7 @@ early_init(int *argc, char **argv) /* } break; } + case 'S' : if (argv[i][2] == 'P') { int ptot, ponln; @@ -922,7 +893,6 @@ early_init(int *argc, char **argv) /* ("using %d:%d scheduler percentages\n", schdlrs_percentage, schdlrs_onln_percentage)); } -#ifdef ERTS_DIRTY_SCHEDULERS else if (argv[i][2] == 'D') { char *arg; char *type = argv[i]+3; @@ -1034,7 +1004,6 @@ early_init(int *argc, char **argv) /* break; } } -#endif else { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -1093,7 +1062,6 @@ early_init(int *argc, char **argv) /* i++; } -#ifdef ERTS_SMP /* apply any scheduler percentages */ if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) { schdlrs = schdlrs * schdlrs_percentage / 100; @@ -1117,12 +1085,6 @@ early_init(int *argc, char **argv) /* erts_usage(); } } -#else - /* Silence gcc warnings */ - (void)schdlrs_percentage; - (void)schdlrs_onln_percentage; -#endif -#ifdef ERTS_DIRTY_SCHEDULERS /* apply any dirty scheduler precentages */ if (dirty_cpu_scheds_pctg != 100 || dirty_cpu_scheds_onln_pctg != 100) { dirty_cpu_scheds = dirty_cpu_scheds * dirty_cpu_scheds_pctg / 100; @@ -1136,33 +1098,25 @@ early_init(int *argc, char **argv) /* dirty_cpu_scheds_online = schdlrs_onln; if (dirty_cpu_scheds_online < 1) dirty_cpu_scheds_online = 1; -#endif } -#ifndef USE_THREADS - erts_async_max_threads = 0; -#endif -#ifdef ERTS_SMP no_schedulers = schdlrs; no_schedulers_online = schdlrs_onln; erts_no_schedulers = (Uint) no_schedulers; -#else - erts_no_schedulers = 1; -#endif -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers = dirty_cpu_scheds; no_dirty_cpu_schedulers_online = dirty_cpu_scheds_online; erts_no_dirty_io_schedulers = no_dirty_io_schedulers = dirty_io_scheds; -#endif erts_early_init_scheduling(no_schedulers); alloc_opts.ncpu = ncpu; erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ /* Require allocators */ -#ifdef ERTS_SMP + + erts_init_check_io(argc, argv); + /* * Thread progress management: * @@ -1170,22 +1124,18 @@ early_init(int *argc, char **argv) /* * ** Scheduler threads (see erl_process.c) * ** Aux thread (see erl_process.c) * ** Sys message dispatcher thread (see erl_trace.c) + * ** IO Poll threads (see erl_check_io.c) * * * Unmanaged threads that need to register: * ** Async threads (see erl_async.c) * ** Dirty scheduler threads */ erts_thr_progress_init(no_schedulers, - no_schedulers+2, -#ifndef ERTS_DIRTY_SCHEDULERS - erts_async_max_threads -#else + no_schedulers+2+erts_no_poll_threads, erts_async_max_threads + erts_no_dirty_cpu_schedulers + erts_no_dirty_io_schedulers -#endif ); -#endif erts_thr_q_init(); erts_init_utils(); erts_early_init_cpu_topology(no_schedulers, @@ -1193,7 +1143,6 @@ early_init(int *argc, char **argv) /* max_reader_groups, &reader_groups); -#ifdef USE_THREADS { erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; elid.mem.std.alloc = ethr_std_alloc; @@ -1210,7 +1159,6 @@ early_init(int *argc, char **argv) /* erts_thr_late_init(&elid); } -#endif erts_msacc_early_init(); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -1237,40 +1185,6 @@ early_init(int *argc, char **argv) /* return ncpu; } -#ifndef ERTS_SMP - -void *erts_scheduler_stack_limit; - - -static void set_main_stack_size(void) -{ - char c; - UWord stacksize; -# if HAVE_DECL_GETRLIMIT && HAVE_DECL_SETRLIMIT && HAVE_DECL_RLIMIT_STACK - struct rlimit rl; - int bytes; - stacksize = erts_sched_thread_suggested_stack_size * sizeof(Uint) * 1024; - /* Add some extra pages... neede by some systems... */ - bytes = (int) stacksize + 3*erts_sys_get_page_size(); - if (getrlimit(RLIMIT_STACK, &rl) != 0 || - (rl.rlim_cur = bytes, setrlimit(RLIMIT_STACK, &rl) != 0)) { - erts_fprintf(stderr, "failed to set stack size for scheduler " - "thread to %d bytes\n", bytes); - erts_usage(); - } -# else - if (modified_sched_thread_suggested_stack_size) { - erts_fprintf(stderr, "no OS support for dynamic stack size limit\n"); - erts_usage(); - } - /* Be conservative and hope it is not more than 64 kWords... */ - stacksize = 64*1024*sizeof(void *); -# endif - - erts_scheduler_stack_limit = erts_calc_stacklimit(&c, stacksize); -} - -#endif void erl_start(int argc, char **argv) @@ -1304,7 +1218,7 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic32_set_nob(&erts_max_gen_gcs, + erts_atomic32_set_nob(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } @@ -1319,10 +1233,8 @@ erl_start(int argc, char **argv) * a lot of stack. */ erts_sched_thread_suggested_stack_size = ERTS_DEFAULT_SCHED_STACK_SIZE; -#ifdef ERTS_DIRTY_SCHEDULERS erts_dcpu_sched_thread_suggested_stack_size = ERTS_DEFAULT_DCPU_SCHED_STACK_SIZE; erts_dio_sched_thread_suggested_stack_size = ERTS_DEFAULT_DIO_SCHED_STACK_SIZE; -#endif #ifdef DEBUG verbose = DEBUG_DEFAULT; @@ -1490,12 +1402,8 @@ erl_start(int argc, char **argv) #ifdef DEBUG strcat(tmp, ",DEBUG"); #endif -#ifdef ERTS_SMP strcat(tmp, ",SMP"); -#endif -#ifdef USE_THREADS strcat(tmp, ",ASYNC_THREADS"); -#endif #ifdef HIPE strcat(tmp, ",HIPE"); #endif @@ -1671,16 +1579,6 @@ erl_start(int argc, char **argv) have_break_handler = 0; break; - case 'K': - /* If kernel poll support is present, - erl_sys_args() will remove the K parameter - and value */ - get_arg(argv[i]+2, argv[i+1], &i); - erts_fprintf(stderr, - "kernel-poll not supported; \"K\" parameter ignored\n", - arg); - break; - case 'n': arg = get_arg(argv[i]+2, argv[i+1], &i); switch (arg[0]) { @@ -1850,22 +1748,9 @@ erl_start(int argc, char **argv) erts_usage(); } } - else if (has_prefix("ecio", sub_param)) { - arg = get_arg(sub_param+4, argv[i+1], &i); -#ifndef __OSE__ - if (sys_strcmp("true", arg) == 0) - erts_eager_check_io = 1; - else -#endif - if (sys_strcmp("false", arg) == 0) - erts_eager_check_io = 0; - else { - erts_fprintf(stderr, - "bad schedule eager check I/O value '%s'\n", - arg); - erts_usage(); - } - } + else if (has_prefix("ecio", sub_param)) { + /* ignore argument, eager check io no longer used */ + } else if (has_prefix("pp", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); if (sys_strcmp(arg, "true") == 0) @@ -1941,7 +1826,6 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("scheduler wakeup threshold: %s\n", arg)); } -#ifdef ERTS_DIRTY_SCHEDULERS else if (has_prefix("ssdcpu", sub_param)) { /* suggested stack size (Kilo Words) for dirty CPU scheduler threads */ arg = get_arg(sub_param+6, argv[i+1], &i); @@ -1976,7 +1860,6 @@ erl_start(int argc, char **argv) ("suggested dirty IO scheduler thread stack size %d kilo words\n", erts_dio_sched_thread_suggested_stack_size)); } -#endif else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -2007,9 +1890,7 @@ erl_start(int argc, char **argv) arg); erts_usage(); } -#ifdef ERTS_SMP erts_runq_supervision_interval = val; -#endif } else { erts_fprintf(stderr, "bad scheduling option %s\n", argv[i]); @@ -2293,12 +2174,10 @@ erl_start(int argc, char **argv) if (erts_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; -#ifdef ERTS_DIRTY_SCHEDULERS if (erts_dcpu_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_dcpu_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; if (erts_dio_sched_thread_suggested_stack_size < ERTS_SCHED_THREAD_MIN_STACK_SIZE) erts_dio_sched_thread_suggested_stack_size = ERTS_SCHED_THREAD_MIN_STACK_SIZE; -#endif erl_init(ncpu, proc_tab_sz, @@ -2343,7 +2222,6 @@ erl_start(int argc, char **argv) && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); -#ifdef ERTS_DIRTY_SCHEDULERS pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker", !0); erts_dirty_process_code_checker = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, @@ -2351,35 +2229,20 @@ erl_start(int argc, char **argv) ASSERT(erts_dirty_process_code_checker && erts_dirty_process_code_checker->common.id == pid); erts_proc_inc_refc(erts_dirty_process_code_checker); -#endif } -#ifdef ERTS_SMP erts_start_schedulers(); - /* Let system specific code decide what to do with the main thread... */ - erts_sys_main_thread(); /* May or may not return! */ -#else - { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - erts_msacc_init_thread("scheduler", 1, 1); - erts_thr_set_main_status(1, 1); -#if ERTS_USE_ASYNC_READY_Q - esdp->aux_work_data.async_ready.queue - = erts_get_async_ready_queue(1); -#endif - set_main_stack_size(); - erts_sched_init_time_sup(esdp); - erts_ets_sched_spec_data_init(esdp); - erts_aux_work_timeout_late_init(esdp); - process_main(esdp->x_reg_array, esdp->f_reg_array); - } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_post_startup(); #endif + + /* Let system specific code decide what to do with the main thread... */ + erts_sys_main_thread(); /* May or may not return! */ } -#ifdef USE_THREADS __decl_noreturn void erts_thr_fatal_error(int err, char *what) { @@ -2393,7 +2256,6 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what) abort(); } -#endif static void system_cleanup(int flush_async) @@ -2406,7 +2268,6 @@ system_cleanup(int flush_async) * Another thread is currently exiting the system; * wait for it to do its job. */ -#ifdef ERTS_SMP if (erts_thr_progress_is_managed_thread()) { /* * The exiting thread might be waiting for @@ -2415,7 +2276,6 @@ system_cleanup(int flush_async) erts_thr_progress_active(NULL, 0); erts_thr_progress_prepare_wait(NULL); } -#endif /* Wait forever... */ while (1) erts_milli_sleep(10000000); @@ -2430,17 +2290,12 @@ system_cleanup(int flush_async) if (!flush_async || !erts_initialized -#if defined(USE_THREADS) && !defined(ERTS_SMP) - || !erts_equal_tids(erts_main_thread, erts_thr_self()) -#endif ) return; -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); #endif -#endif erts_exit_flush_async(); } diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index 4d4defd8b5..634509f880 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1200,7 +1200,8 @@ erts_instr_init(int stat, int map_stat) stats = erts_alloc(ERTS_ALC_T_INSTR_INFO, sizeof(struct stats_)); - erts_mtx_init(&instr_mutex, "instr"); + erts_mtx_init(&instr_mutex, "instr", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); mem_anchor = NULL; @@ -1223,7 +1224,8 @@ erts_instr_init(int stat, int map_stat) if (map_stat) { - erts_mtx_init(&instr_x_mutex, "instr_x"); + erts_mtx_init(&instr_x_mutex, "instr_x", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); erts_instr_memory_map = 1; erts_instr_stat = 1; diff --git a/erts/emulator/beam/erl_io_queue.c b/erts/emulator/beam/erl_io_queue.c new file mode 100644 index 0000000000..190ba6bbb9 --- /dev/null +++ b/erts/emulator/beam/erl_io_queue.c @@ -0,0 +1,1231 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" + +#define ERL_WANT_HIPE_BIF_WRAPPER__ +#include "bif.h" +#undef ERL_WANT_HIPE_BIF_WRAPPER__ + +#include "erl_bits.h" +#include "erl_io_queue.h" + +#define IOL2V_SMALL_BIN_LIMIT (ERL_ONHEAP_BIN_LIMIT * 4) + +static void free_binary(ErtsIOQBinary *b, int driver); +static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver); + +void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver) +{ + + ERTS_CT_ASSERT(offsetof(ErlNifIOVec,flags) == sizeof(ErtsIOVecCommon)); + ERTS_CT_ASSERT(sizeof(ErlIOVec) == sizeof(ErtsIOVecCommon)); + ERTS_CT_ASSERT(sizeof(size_t) == sizeof(ErlDrvSizeT)); + ERTS_CT_ASSERT(sizeof(size_t) == sizeof(Uint)); + + q->alct = alct; + q->driver = driver; + q->size = 0; + q->v_head = q->v_tail = q->v_start = q->v_small; + q->v_end = q->v_small + ERTS_SMALL_IO_QUEUE; + q->b_head = q->b_tail = q->b_start = q->b_small; + q->b_end = q->b_small + ERTS_SMALL_IO_QUEUE; +} + +void erts_ioq_clear(ErtsIOQueue *q) +{ + ErtsIOQBinary** binp = q->b_head; + int driver = q->driver; + + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + + while(binp < q->b_tail) { + if (*binp != NULL) + free_binary(*binp, driver); + binp++; + } + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->v_start = q->v_end = q->v_head = q->v_tail = NULL; + q->b_start = q->b_end = q->b_head = q->b_tail = NULL; + q->size = 0; +} + +static void free_binary(ErtsIOQBinary *b, int driver) +{ + if (driver) + driver_free_binary(&b->driver); + else if (erts_refc_dectest(&b->nif.intern.refc, 0) == 0) + erts_bin_free(&b->nif); +} + +static ErtsIOQBinary *alloc_binary(Uint size, char *source, void **iov_base, int driver) +{ + if (driver) { + ErlDrvBinary *bin = driver_alloc_binary(size); + if (!bin) return NULL; + sys_memcpy(bin->orig_bytes, source, size); + *iov_base = bin->orig_bytes; + return (ErtsIOQBinary *)bin; + } else { + /* This clause can be triggered in enif_ioq_enq_binary is used */ + Binary *bin = erts_bin_nrml_alloc(size); + if (!bin) return NULL; + erts_refc_init(&bin->intern.refc, 1); + sys_memcpy(bin->orig_bytes, source, size); + *iov_base = bin->orig_bytes; + return (ErtsIOQBinary *)bin; + } +} + +Uint erts_ioq_size(ErtsIOQueue *q) +{ + return q->size; +} + +/* expand queue to hold n elements in tail or head */ +static int expandq(ErtsIOQueue* q, int n, int tail) +/* tail: 0 if make room in head, make room in tail otherwise */ +{ + int h_sz; /* room before header */ + int t_sz; /* room after tail */ + int q_sz; /* occupied */ + int nvsz; + SysIOVec* niov; + ErtsIOQBinary** nbinv; + + h_sz = q->v_head - q->v_start; + t_sz = q->v_end - q->v_tail; + q_sz = q->v_tail - q->v_head; + + if (tail && (n <= t_sz)) /* do we need to expand tail? */ + return 0; + else if (!tail && (n <= h_sz)) /* do we need to expand head? */ + return 0; + else if (n > (h_sz + t_sz)) { /* need to allocate */ + /* we may get little extra but it ok */ + nvsz = (q->v_end - q->v_start) + n; + + niov = erts_alloc_fnf(q->alct, nvsz * sizeof(SysIOVec)); + if (!niov) + return -1; + nbinv = erts_alloc_fnf(q->alct, nvsz * sizeof(ErtsIOQBinary**)); + if (!nbinv) { + erts_free(q->alct, (void *) niov); + return -1; + } + if (tail) { + sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec)); + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + q->v_start = niov; + q->v_end = niov + nvsz; + q->v_head = q->v_start; + q->v_tail = q->v_head + q_sz; + + sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->b_start = nbinv; + q->b_end = nbinv + nvsz; + q->b_head = q->b_start; + q->b_tail = q->b_head + q_sz; + } + else { + sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); + if (q->v_start != q->v_small) + erts_free(q->alct, (void *) q->v_start); + q->v_start = niov; + q->v_end = niov + nvsz; + q->v_tail = q->v_end; + q->v_head = q->v_tail - q_sz; + + sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + if (q->b_start != q->b_small) + erts_free(q->alct, (void *) q->b_start); + q->b_start = nbinv; + q->b_end = nbinv + nvsz; + q->b_tail = q->b_end; + q->b_head = q->b_tail - q_sz; + } + } + else if (tail) { /* move to beginning to make room in tail */ + sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec)); + q->v_head = q->v_start; + q->v_tail = q->v_head + q_sz; + sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + q->b_head = q->b_start; + q->b_tail = q->b_head + q_sz; + } + else { /* move to end to make room */ + sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); + q->v_tail = q->v_end; + q->v_head = q->v_tail-q_sz; + sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErtsIOQBinary*)); + q->b_tail = q->b_end; + q->b_head = q->b_tail-q_sz; + } + + return 0; +} + +static +int skip(ErtsIOVec* vec, Uint skipbytes, + SysIOVec **iovp, ErtsIOQBinary ***binvp, + Uint *lenp) +{ + int n; + Uint len; + SysIOVec* iov; + ErtsIOQBinary** binv; + + if (vec->common.size <= skipbytes) + return -1; + + iov = vec->common.iov; + binv = vec->common.binv; + n = vec->common.vsize; + /* we use do here to strip iov_len=0 from beginning */ + do { + len = iov->iov_len; + if (len <= skipbytes) { + skipbytes -= len; + iov++; + binv++; + n--; + } + else { + iov->iov_base = ((char *)(iov->iov_base)) + skipbytes; + iov->iov_len -= skipbytes; + skipbytes = 0; + } + } while(skipbytes > 0); + + *binvp = binv; + *iovp = iov; + *lenp = len; + + return n; +} + +/* Put elements from vec at q tail */ +int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *eiov, Uint skipbytes) +{ + int n; + Uint len; + Uint size = eiov->common.size - skipbytes; + SysIOVec *iov; + ErtsIOQBinary** binv; + ErtsIOQBinary* b; + + if (q == NULL) + return -1; + + ASSERT(eiov->common.size >= skipbytes); + if (eiov->common.size <= skipbytes) + return 0; + + n = skip(eiov, skipbytes, &iov, &binv, &len); + + if (n < 0) + return n; + + if (q->v_tail + n >= q->v_end) + if (expandq(q, n, 1)) + return -1; + + /* Queue and reference all binaries (remove zero length items) */ + while(n--) { + if ((len = iov->iov_len) > 0) { + if ((b = *binv) == NULL) { /* special case create binary ! */ + b = alloc_binary(len, iov->iov_base, (void**)&q->v_tail->iov_base, + q->driver); + if (!b) return -1; + *q->b_tail++ = b; + q->v_tail->iov_len = len; + q->v_tail++; + } + else { + if (q->driver) + driver_binary_inc_refc(&b->driver); + else + erts_refc_inc(&b->nif.intern.refc, 1); + *q->b_tail++ = b; + *q->v_tail++ = *iov; + } + } + iov++; + binv++; + } + q->size += size; /* update total size in queue */ + return 0; +} + +/* Put elements from vec at q head */ +int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec* vec, Uint skipbytes) +{ + int n; + Uint len; + Uint size = vec->common.size - skipbytes; + SysIOVec* iov; + ErtsIOQBinary** binv; + ErtsIOQBinary* b; + + if (q == NULL) + return -1; + + ASSERT(vec->common.size >= skipbytes); + if (vec->common.size <= skipbytes) + return 0; + + n = skip(vec, skipbytes, &iov, &binv, &len); + + if (n < 0) + return n; + + if (q->v_head - n < q->v_start) + if (expandq(q, n, 0)) + return -1; + + /* Queue and reference all binaries (remove zero length items) */ + iov += (n-1); /* move to end */ + binv += (n-1); /* move to end */ + while(n--) { + if ((len = iov->iov_len) > 0) { + if ((b = *binv) == NULL) { /* special case create binary ! */ + if (q->driver) { + ErlDrvBinary *bin = driver_alloc_binary(len); + if (!bin) return -1; + sys_memcpy(bin->orig_bytes, iov->iov_base, len); + b = (ErtsIOQBinary *)bin; + q->v_head->iov_base = bin->orig_bytes; + } + *--q->b_head = b; + q->v_head--; + q->v_head->iov_len = len; + } + else { + if (q->driver) + driver_binary_inc_refc(&b->driver); + else + erts_refc_inc(&b->nif.intern.refc, 1); + *--q->b_head = b; + *--q->v_head = *iov; + } + } + iov--; + binv--; + } + q->size += size; /* update total size in queue */ + return 0; +} + + +/* +** Remove size bytes from queue head +** Return number of bytes that remain in queue +*/ +int erts_ioq_deq(ErtsIOQueue *q, Uint size) +{ + Uint len; + + if ((q == NULL) || (q->size < size)) + return -1; + q->size -= size; + while (size > 0) { + ASSERT(q->v_head != q->v_tail); + + len = q->v_head->iov_len; + if (len <= size) { + size -= len; + free_binary(*q->b_head, q->driver); + *q->b_head++ = NULL; + q->v_head++; + } + else { + q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; + q->v_head->iov_len -= size; + size = 0; + } + } + + /* restart pointers (optimised for enq) */ + if (q->v_head == q->v_tail) { + q->v_head = q->v_tail = q->v_start; + q->b_head = q->b_tail = q->b_start; + } + return 0; +} + + +Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev) { + ASSERT(ev); + + if (! q) { + return (Uint) -1; + } else { + if ((ev->common.vsize = q->v_tail - q->v_head) == 0) { + ev->common.size = 0; + ev->common.iov = NULL; + ev->common.binv = NULL; + } else { + ev->common.size = q->size; + ev->common.iov = q->v_head; + ev->common.binv = q->b_head; + } + return q->size; + } +} + +SysIOVec* erts_ioq_peekq(ErtsIOQueue *q, int* vlenp) /* length of io-vector */ +{ + + if (q == NULL) { + *vlenp = -1; + return NULL; + } + if ((*vlenp = (q->v_tail - q->v_head)) == 0) + return NULL; + return q->v_head; +} + +/* Fills a possibly deep list of chars and binaries into vec +** Small characters are first stored in the buffer buf of length ln +** binaries found are copied and linked into msoh +** Return vector length on succsess, +** -1 on overflow +** -2 on type error +*/ + +static ERTS_INLINE void +io_list_to_vec_set_vec(SysIOVec **iov, ErtsIOQBinary ***binv, + ErtsIOQBinary *bin, byte *ptr, Uint len, + int *vlen) +{ + while (len > MAX_SYSIOVEC_IOVLEN) { + (*iov)->iov_base = ptr; + (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN; + ptr += MAX_SYSIOVEC_IOVLEN; + len -= MAX_SYSIOVEC_IOVLEN; + (*iov)++; + (*vlen)++; + *(*binv)++ = bin; + } + (*iov)->iov_base = ptr; + (*iov)->iov_len = len; + *(*binv)++ = bin; + (*iov)++; + (*vlen)++; +} + +int +erts_ioq_iolist_to_vec(Eterm obj, /* io-list */ + SysIOVec* iov, /* io vector */ + ErtsIOQBinary** binv, /* binary reference vector */ + ErtsIOQBinary* cbin, /* binary to store characters */ + Uint bin_limit, /* small binaries limit */ + int driver) +{ + DECLARE_ESTACK(s); + Eterm* objp; + byte *buf = NULL; + Uint len = 0; + Uint csize = 0; + int vlen = 0; + byte* cptr; + + if (cbin) { + if (driver) { + buf = (byte*)cbin->driver.orig_bytes; + len = cbin->driver.orig_size; + } else { + buf = (byte*)cbin->nif.orig_bytes; + len = cbin->nif.orig_size; + } + } + cptr = buf; + + goto L_jump_start; /* avoid push */ + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_jump_start: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + obj = CAR(objp); + if (is_byte(obj)) { + if (len == 0) + goto L_overflow; + *buf++ = unsigned_val(obj); + csize++; + len--; + } else if (is_binary(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto handle_binary; + } else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto L_iter_list; /* on head */ + } else if (!is_nil(obj)) { + goto L_type_error; + } + obj = CDR(objp); + if (is_list(obj)) + goto L_iter_list; /* on tail */ + else if (is_binary(obj)) { + goto handle_binary; + } else if (!is_nil(obj)) { + goto L_type_error; + } + } else if (is_binary(obj)) { + Eterm real_bin; + Uint offset; + Eterm* bptr; + Uint size; + int bitoffs; + int bitsize; + + handle_binary: + size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + ASSERT(bitsize == 0); + bptr = binary_val(real_bin); + if (*bptr == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin *) bptr; + if (bitoffs != 0) { + if (len < size) { + goto L_overflow; + } + erts_copy_bits(pb->bytes+offset, bitoffs, 1, + (byte *) buf, 0, 1, size*8); + csize += size; + buf += size; + len -= size; + } else if (bin_limit && size < bin_limit) { + if (len < size) { + goto L_overflow; + } + sys_memcpy(buf, pb->bytes+offset, size); + csize += size; + buf += size; + len -= size; + } else { + ErtsIOQBinary *qbin; + if (csize != 0) { + io_list_to_vec_set_vec(&iov, &binv, cbin, + cptr, csize, &vlen); + cptr = buf; + csize = 0; + } + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + if (driver) + qbin = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val); + else + qbin = (ErtsIOQBinary*)pb->val; + + io_list_to_vec_set_vec( + &iov, &binv, qbin, + pb->bytes+offset, size, &vlen); + } + } else { + ErlHeapBin* hb = (ErlHeapBin *) bptr; + if (len < size) { + goto L_overflow; + } + copy_binary_to_buffer(buf, 0, + ((byte *) hb->data)+offset, bitoffs, + 8*size); + csize += size; + buf += size; + len -= size; + } + } else if (!is_nil(obj)) { + goto L_type_error; + } + } + + if (csize != 0) { + io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen); + } + + DESTROY_ESTACK(s); + return vlen; + + L_type_error: + DESTROY_ESTACK(s); + return -2; + + L_overflow: + DESTROY_ESTACK(s); + return -1; +} + +static ERTS_INLINE int +io_list_vec_count(Eterm obj, Uint *v_size, + Uint *c_size, Uint *b_size, Uint *in_clist, + Uint *p_v_size, Uint *p_c_size, Uint *p_in_clist, + Uint blimit) +{ + Uint size = binary_size(obj); + Eterm real; + ERTS_DECLARE_DUMMY(Uint offset); + int bitoffs; + int bitsize; + ERTS_GET_REAL_BIN(obj, real, offset, bitoffs, bitsize); + if (bitsize != 0) return 1; + if (thing_subtag(*binary_val(real)) == REFC_BINARY_SUBTAG && + bitoffs == 0) { + *b_size += size; + if (*b_size < size) return 2; + *in_clist = 0; + ++*v_size; + /* If iov_len is smaller then Uint we split the binary into*/ + /* multiple smaller (2GB) elements in the iolist.*/ + *v_size += size / MAX_SYSIOVEC_IOVLEN; + if (size >= blimit) { + *p_in_clist = 0; + ++*p_v_size; + } else { + *p_c_size += size; + if (!*p_in_clist) { + *p_in_clist = 1; + ++*p_v_size; + } + } + } else { + *c_size += size; + if (*c_size < size) return 2; + if (!*in_clist) { + *in_clist = 1; + ++*v_size; + } + *p_c_size += size; + if (!*p_in_clist) { + *p_in_clist = 1; + ++*p_v_size; + } + } + return 0; +} + +#define IO_LIST_VEC_COUNT(obj) \ + do { \ + switch (io_list_vec_count(obj, &v_size, &c_size, \ + &b_size, &in_clist, \ + &p_v_size, &p_c_size, &p_in_clist, \ + blimit)) { \ + case 1: goto L_type_error; \ + case 2: goto L_overflow_error; \ + default: break; \ + } \ + } while(0) + +/* + * Returns 0 if successful and a non-zero value otherwise. + * + * Return values through pointers: + * *vsize - SysIOVec size needed for a writev + * *csize - Number of bytes not in binary (in the common binary) + * *pvsize - SysIOVec size needed if packing small binaries + * *pcsize - Number of bytes in the common binary if packing + * *total_size - Total size of iolist in bytes + */ +int +erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit) +{ + DECLARE_ESTACK(s); + Eterm* objp; + Uint v_size = 0; + Uint c_size = 0; + Uint b_size = 0; + Uint in_clist = 0; + Uint p_v_size = 0; + Uint p_c_size = 0; + Uint p_in_clist = 0; + Uint total; + + goto L_jump_start; /* avoid a push */ + + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + L_jump_start: + if (is_list(obj)) { + L_iter_list: + objp = list_val(obj); + obj = CAR(objp); + + if (is_byte(obj)) { + c_size++; + if (c_size == 0) { + goto L_overflow_error; + } + if (!in_clist) { + in_clist = 1; + v_size++; + } + p_c_size++; + if (!p_in_clist) { + p_in_clist = 1; + p_v_size++; + } + } + else if (is_binary(obj)) { + IO_LIST_VEC_COUNT(obj); + } + else if (is_list(obj)) { + ESTACK_PUSH(s, CDR(objp)); + goto L_iter_list; /* on head */ + } + else if (!is_nil(obj)) { + goto L_type_error; + } + + obj = CDR(objp); + if (is_list(obj)) + goto L_iter_list; /* on tail */ + else if (is_binary(obj)) { /* binary tail is OK */ + IO_LIST_VEC_COUNT(obj); + } + else if (!is_nil(obj)) { + goto L_type_error; + } + } + else if (is_binary(obj)) { + IO_LIST_VEC_COUNT(obj); + } + else if (!is_nil(obj)) { + goto L_type_error; + } + } + + total = c_size + b_size; + if (total < c_size) { + goto L_overflow_error; + } + *total_size = total; + + DESTROY_ESTACK(s); + *vsize = v_size; + *csize = c_size; + *pvsize = p_v_size; + *pcsize = p_c_size; + return 0; + + L_type_error: + L_overflow_error: + DESTROY_ESTACK(s); + return 1; +} + +typedef struct { + Eterm result_head; + Eterm result_tail; + Eterm input_list; + + UWord acc_size; + Binary *acc; + + /* We yield after copying this many bytes into the accumulator (Minus + * eating a few on consing etc). Large binaries will only count to the + * extent their split (if any) resulted in a copy op. */ + UWord bytereds_available; + UWord bytereds_spent; + + Process *process; + ErtsEStack estack; + + Eterm magic_reference; +} iol2v_state_t; + +static int iol2v_state_destructor(Binary *data) { + iol2v_state_t *state = ERTS_MAGIC_BIN_UNALIGNED_DATA(data); + + DESTROY_SAVED_ESTACK(&state->estack); + + if (state->acc != NULL) { + erts_bin_free(state->acc); + } + + return 1; +} + +static void iol2v_init(iol2v_state_t *state, Process *process, Eterm input) { + state->process = process; + + state->result_head = NIL; + state->result_tail = NIL; + state->input_list = input; + + state->magic_reference = NIL; + state->acc_size = 0; + state->acc = NULL; + + CLEAR_SAVED_ESTACK(&state->estack); +} + +static Eterm iol2v_make_sub_bin(iol2v_state_t *state, Eterm bin_term, + UWord offset, UWord size) { + Uint byte_offset, bit_offset, bit_size; + ErlSubBin *sb; + Eterm orig_pb_term; + + sb = (ErlSubBin*)HAlloc(state->process, ERL_SUB_BIN_SIZE); + + ERTS_GET_REAL_BIN(bin_term, orig_pb_term, + byte_offset, bit_offset, bit_size); + + (void)bit_offset; + (void)bit_size; + + sb->thing_word = HEADER_SUB_BIN; + sb->bitsize = 0; + sb->bitoffs = 0; + sb->orig = orig_pb_term; + sb->is_writable = 0; + + sb->offs = byte_offset + offset; + sb->size = size; + + return make_binary(sb); +} + +static Eterm iol2v_promote_acc(iol2v_state_t *state) { + ProcBin *pb; + + state->acc = erts_bin_realloc(state->acc, state->acc_size); + + pb = (ProcBin*)HAlloc(state->process, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = state->acc_size; + pb->val = state->acc; + pb->bytes = (byte*)(state->acc)->orig_bytes; + pb->flags = 0; + pb->next = MSO(state->process).first; + OH_OVERHEAD(&(MSO(state->process)), pb->size / sizeof(Eterm)); + MSO(state->process).first = (struct erl_off_heap_header*)pb; + + state->acc_size = 0; + state->acc = NULL; + + return make_binary(pb); +} + +/* Destructively enqueues a term to the result list, saving us the hassle of + * having to reverse it later. This is safe since GC is disabled and we never + * leak the unfinished term to the outside. */ +static void iol2v_enqueue_result(iol2v_state_t *state, Eterm term) { + Eterm prev_tail; + Eterm *hp; + + prev_tail = state->result_tail; + + hp = HAlloc(state->process, 2); + state->result_tail = CONS(hp, term, NIL); + + if(prev_tail != NIL) { + Eterm *prev_cell = list_val(prev_tail); + CDR(prev_cell) = state->result_tail; + } else { + state->result_head = state->result_tail; + } + + state->bytereds_spent += 1; +} + +#ifndef DEBUG + #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 32) +#else + #define ACC_REALLOCATION_LIMIT (IOL2V_SMALL_BIN_LIMIT * 4) +#endif + +static void iol2v_expand_acc(iol2v_state_t *state, UWord extra) { + UWord required_bytes, acc_alloc_size; + + ERTS_CT_ASSERT(ERTS_UWORD_MAX > ACC_REALLOCATION_LIMIT / 2); + ASSERT(extra >= 1); + + acc_alloc_size = state->acc != NULL ? (state->acc)->orig_size : 0; + required_bytes = state->acc_size + extra; + + if (state->acc == NULL) { + UWord new_size = MAX(required_bytes, IOL2V_SMALL_BIN_LIMIT); + + state->acc = erts_bin_nrml_alloc(new_size); + } else if (required_bytes > acc_alloc_size) { + Binary *prev_acc; + UWord new_size; + + if (acc_alloc_size >= ACC_REALLOCATION_LIMIT) { + /* We skip reallocating once we hit a certain point; it often + * results in extra copying and we're very likely to overallocate + * on anything other than absurdly long byte/heapbin sequences. */ + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + iol2v_expand_acc(state, extra); + return; + } + + new_size = MAX(required_bytes, acc_alloc_size * 2); + prev_acc = state->acc; + + state->acc = erts_bin_realloc(prev_acc, new_size); + + if (prev_acc != state->acc) { + state->bytereds_spent += state->acc_size; + } + } + + state->bytereds_spent += extra; +} + +static int iol2v_append_byte_seq(iol2v_state_t *state, Eterm seq_start, Eterm *seq_end) { + Eterm lookahead, iterator; + Uint observed_bits; + SWord seq_length; + char *acc_data; + + lookahead = seq_start; + seq_length = 0; + + ASSERT(state->bytereds_available > state->bytereds_spent); + + while (is_list(lookahead)) { + Eterm *cell = list_val(lookahead); + + if (!is_small(CAR(cell))) { + break; + } + + if (seq_length * 2 >= (state->bytereds_available - state->bytereds_spent)) { + break; + } + + lookahead = CDR(cell); + seq_length += 1; + } + + ASSERT(seq_length >= 1); + + iol2v_expand_acc(state, seq_length); + + /* Bump a few extra reductions to account for list traversal. */ + state->bytereds_spent += seq_length; + + acc_data = &(state->acc)->orig_bytes[state->acc_size]; + state->acc_size += seq_length; + + iterator = seq_start; + observed_bits = 0; + + while (iterator != lookahead) { + Eterm *cell; + Uint byte; + + cell = list_val(iterator); + iterator = CDR(cell); + + byte = unsigned_val(CAR(cell)); + observed_bits |= byte; + + ASSERT(acc_data < &(state->acc)->orig_bytes[state->acc_size]); + *(acc_data++) = byte; + } + + if (observed_bits > UCHAR_MAX) { + return 0; + } + + ASSERT(acc_data == &(state->acc)->orig_bytes[state->acc_size]); + *seq_end = iterator; + + return 1; +} + +static int iol2v_append_binary(iol2v_state_t *state, Eterm bin_term) { + int is_acc_small, is_bin_small; + UWord combined_size; + UWord binary_size; + + Uint byte_offset, bit_offset, bit_size; + byte *binary_data; + + Eterm *parent_header; + Eterm parent_binary; + + ASSERT(state->bytereds_available > state->bytereds_spent); + + ERTS_GET_REAL_BIN(bin_term, parent_binary, byte_offset, bit_offset, bit_size); + parent_header = binary_val(parent_binary); + binary_size = binary_size(bin_term); + + if (bit_offset != 0 || bit_size != 0) { + return 0; + } else if (binary_size == 0) { + state->bytereds_spent += 1; + return 1; + } + + is_acc_small = state->acc_size < IOL2V_SMALL_BIN_LIMIT; + is_bin_small = binary_size < IOL2V_SMALL_BIN_LIMIT; + combined_size = binary_size + state->acc_size; + + if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin*)parent_header; + + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + + binary_data = &((byte*)pb->bytes)[byte_offset]; + } else { + ErlHeapBin *hb = (ErlHeapBin*)parent_header; + + ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG); + ASSERT(is_bin_small); + + binary_data = &((byte*)&hb->data)[byte_offset]; + } + + if (!is_bin_small && (state->acc_size == 0 || !is_acc_small)) { + /* Avoid combining if we encounter an acceptably large binary while the + * accumulator is either empty or large enough to be returned on its + * own. */ + if (state->acc_size != 0) { + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + } + + iol2v_enqueue_result(state, bin_term); + } else if (is_bin_small || combined_size < (IOL2V_SMALL_BIN_LIMIT * 2)) { + /* If the candidate is small or we can't split the combination in two, + * then just copy it into the accumulator. */ + iol2v_expand_acc(state, binary_size); + + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, binary_size); + + state->acc_size += binary_size; + } else { + /* Otherwise, append enough data for the accumulator to be valid, and + * then return the rest as a sub-binary. */ + UWord spill = IOL2V_SMALL_BIN_LIMIT - state->acc_size; + Eterm binary_tail; + + iol2v_expand_acc(state, spill); + + sys_memcpy(&(state->acc)->orig_bytes[state->acc_size], + binary_data, spill); + + state->acc_size += spill; + + binary_tail = iol2v_make_sub_bin(state, bin_term, spill, + binary_size - spill); + + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + iol2v_enqueue_result(state, binary_tail); + } + + return 1; +} + +static BIF_RETTYPE iol2v_yield(iol2v_state_t *state) { + if (is_nil(state->magic_reference)) { + iol2v_state_t *boxed_state; + Binary *magic_binary; + Eterm *hp; + + magic_binary = erts_create_magic_binary_x(sizeof(*state), + &iol2v_state_destructor, ERTS_ALC_T_BINARY, 1); + + boxed_state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic_binary); + sys_memcpy(boxed_state, state, sizeof(*state)); + + hp = HAlloc(boxed_state->process, ERTS_MAGIC_REF_THING_SIZE); + boxed_state->magic_reference = + erts_mk_magic_ref(&hp, &MSO(boxed_state->process), magic_binary); + + state = boxed_state; + } + + ERTS_BIF_YIELD1(bif_export[BIF_iolist_to_iovec_1], + state->process, state->magic_reference); +} + +static BIF_RETTYPE iol2v_continue(iol2v_state_t *state) { + Eterm iterator; + + DECLARE_ESTACK(s); + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + state->bytereds_available = + ERTS_BIF_REDS_LEFT(state->process) * IOL2V_SMALL_BIN_LIMIT; + state->bytereds_spent = 0; + + if (state->estack.start) { + ESTACK_RESTORE(s, &state->estack); + } + + iterator = state->input_list; + + for(;;) { + if (state->bytereds_spent >= state->bytereds_available) { + ESTACK_SAVE(s, &state->estack); + state->input_list = iterator; + + return iol2v_yield(state); + } + + while (is_list(iterator)) { + Eterm *cell; + Eterm head; + + cell = list_val(iterator); + head = CAR(cell); + + if (is_binary(head)) { + if (!iol2v_append_binary(state, head)) { + goto l_badarg; + } + + iterator = CDR(cell); + } else if (is_small(head)) { + Eterm seq_end; + + if (!iol2v_append_byte_seq(state, iterator, &seq_end)) { + goto l_badarg; + } + + iterator = seq_end; + } else if (is_list(head) || is_nil(head)) { + Eterm tail = CDR(cell); + + if (!is_nil(tail)) { + ESTACK_PUSH(s, tail); + } + + state->bytereds_spent += 1; + iterator = head; + } else { + goto l_badarg; + } + + if (state->bytereds_spent >= state->bytereds_available) { + ESTACK_SAVE(s, &state->estack); + state->input_list = iterator; + + return iol2v_yield(state); + } + } + + if (is_binary(iterator)) { + if (!iol2v_append_binary(state, iterator)) { + goto l_badarg; + } + } else if (!is_nil(iterator)) { + goto l_badarg; + } + + if(ESTACK_ISEMPTY(s)) { + break; + } + + iterator = ESTACK_POP(s); + } + + if (state->acc_size != 0) { + iol2v_enqueue_result(state, iol2v_promote_acc(state)); + } + + BUMP_REDS(state->process, state->bytereds_spent / IOL2V_SMALL_BIN_LIMIT); + + CLEAR_SAVED_ESTACK(&state->estack); + DESTROY_ESTACK(s); + + BIF_RET(state->result_head); + +l_badarg: + CLEAR_SAVED_ESTACK(&state->estack); + DESTROY_ESTACK(s); + + if (state->acc != NULL) { + erts_bin_free(state->acc); + state->acc = NULL; + } + + BIF_ERROR(state->process, BADARG); +} + +HIPE_WRAPPER_BIF_DISABLE_GC(iolist_to_iovec, 1) + +BIF_RETTYPE iolist_to_iovec_1(BIF_ALIST_1) { + BIF_RETTYPE result; + + if (is_nil(BIF_ARG_1)) { + BIF_RET(NIL); + } else if (is_binary(BIF_ARG_1)) { + if (binary_size(BIF_ARG_1) != 0) { + Eterm *hp = HAlloc(BIF_P, 2); + + BIF_RET(CONS(hp, BIF_ARG_1, NIL)); + } else { + BIF_RET(NIL); + } + } else if (is_internal_magic_ref(BIF_ARG_1)) { + iol2v_state_t *state; + Binary *magic; + + magic = erts_magic_ref2bin(BIF_ARG_1); + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != &iol2v_state_destructor) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } + + ASSERT(BIF_P->flags & F_DISABLE_GC); + + state = ERTS_MAGIC_BIN_UNALIGNED_DATA(magic); + result = iol2v_continue(state); + } else if (!is_list(BIF_ARG_1)) { + ASSERT(!(BIF_P->flags & F_DISABLE_GC)); + BIF_ERROR(BIF_P, BADARG); + } else { + iol2v_state_t state; + + iol2v_init(&state, BIF_P, BIF_ARG_1); + + erts_set_gc_state(BIF_P, 0); + + result = iol2v_continue(&state); + } + + if (result != THE_NON_VALUE || BIF_P->freason != TRAP) { + erts_set_gc_state(BIF_P, 1); + } + + BIF_RET(result); +} diff --git a/erts/emulator/beam/erl_io_queue.h b/erts/emulator/beam/erl_io_queue.h new file mode 100644 index 0000000000..51abe99510 --- /dev/null +++ b/erts/emulator/beam/erl_io_queue.h @@ -0,0 +1,201 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A queue used for storing binary data that should be + * passed to writev or similar functions. Used by both + * the nif and driver api. + * + * Author: Lukas Larsson + */ + +#ifndef ERL_IO_QUEUE_H__TYPES__ +#define ERL_IO_QUEUE_H__TYPES__ + +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ +#include "erl_nif.h" + +#ifdef DEBUG +#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1)) +#else +#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1)) +#endif + +#define ERTS_SMALL_IO_QUEUE 5 + +typedef union { + ErlDrvBinary driver; + Binary nif; +} ErtsIOQBinary; + +typedef struct { + int vsize; /* length of vectors */ + Uint size; /* total size in bytes */ + SysIOVec* iov; + ErtsIOQBinary** binv; +} ErtsIOVecCommon; + +typedef union { + ErtsIOVecCommon common; + ErlIOVec driver; + ErlNifIOVec nif; +} ErtsIOVec; + +/* head/tail represent the data in the queue + * start/end represent the edges of the allocated queue + * small is used when the number of iovec elements is < SMALL_IO_QUEUE + */ +typedef struct erts_io_queue { + ErtsAlcType_t alct; + int driver; + Uint size; /* total size in bytes */ + + SysIOVec* v_start; + SysIOVec* v_end; + SysIOVec* v_head; + SysIOVec* v_tail; + SysIOVec v_small[ERTS_SMALL_IO_QUEUE]; + + ErtsIOQBinary **b_start; + ErtsIOQBinary **b_end; + ErtsIOQBinary **b_head; + ErtsIOQBinary **b_tail; + ErtsIOQBinary *b_small[ERTS_SMALL_IO_QUEUE]; + +} ErtsIOQueue; + +#endif /* ERL_IO_QUEUE_H__TYPES__ */ + +#if !defined(ERL_IO_QUEUE_H) && !defined(ERTS_IO_QUEUE_TYPES_ONLY__) +#define ERL_IO_QUEUE_H + +#include "erl_binary.h" +#include "erl_bits.h" + +void erts_ioq_init(ErtsIOQueue *q, ErtsAlcType_t alct, int driver); +void erts_ioq_clear(ErtsIOQueue *q); +Uint erts_ioq_size(ErtsIOQueue *q); +int erts_ioq_enqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip); +int erts_ioq_pushqv(ErtsIOQueue *q, ErtsIOVec *vec, Uint skip); +int erts_ioq_deq(ErtsIOQueue *q, Uint Uint); +Uint erts_ioq_peekqv(ErtsIOQueue *q, ErtsIOVec *ev); +SysIOVec *erts_ioq_peekq(ErtsIOQueue *q, int *vlenp); +Uint erts_ioq_sizeq(ErtsIOQueue *q); + +int erts_ioq_iolist_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit); +int erts_ioq_iolist_to_vec(Eterm obj, SysIOVec* iov, + ErtsIOQBinary** binv, ErtsIOQBinary* cbin, + Uint bin_limit, int driver_binary); + +ERTS_GLB_INLINE +int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit); +ERTS_GLB_INLINE +int erts_ioq_iodata_to_vec(Eterm obj, SysIOVec* iov, + ErtsIOQBinary** binv, ErtsIOQBinary* cbin, + Uint bin_limit, int driver_binary); + + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +int erts_ioq_iodata_vec_len(Eterm obj, int* vsize, Uint* csize, + Uint* pvsize, Uint* pcsize, + Uint* total_size, Uint blimit) { + if (is_binary(obj)) { + /* We optimize for when we get a procbin without a bit-offset + * that fits in one iov slot + */ + Eterm real_bin; + byte bitoffs; + byte bitsize; + ERTS_DECLARE_DUMMY(Uint offset); + Uint size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) { + *vsize = 1; + *pvsize = 1; + if (thing_subtag(*binary_val(real_bin)) == REFC_BINARY_SUBTAG) { + *csize = 0; + *pcsize = 0; + } else { + *csize = size; + *pcsize = size; + } + *total_size = size; + return 0; + } + } + + return erts_ioq_iolist_vec_len(obj, vsize, csize, + pvsize, pcsize, total_size, blimit); +} + +ERTS_GLB_INLINE +int erts_ioq_iodata_to_vec(Eterm obj, + SysIOVec *iov, + ErtsIOQBinary **binv, + ErtsIOQBinary *cbin, + Uint bin_limit, + int driver) +{ + if (is_binary(obj)) { + Eterm real_bin; + byte bitoffs; + byte bitsize; + Uint offset; + Uint size = binary_size(obj); + ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + if (size < MAX_SYSIOVEC_IOVLEN && bitoffs == 0 && bitsize == 0) { + Eterm *bptr = binary_val(real_bin); + if (thing_subtag(*bptr) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin *)bptr; + if (pb->flags) + erts_emasculate_writable_binary(pb); + iov[0].iov_base = pb->bytes+offset; + iov[0].iov_len = size; + if (driver) + binv[0] = (ErtsIOQBinary*)Binary2ErlDrvBinary(pb->val); + else + binv[0] = (ErtsIOQBinary*)pb->val; + return 1; + } else { + ErlHeapBin* hb = (ErlHeapBin *)bptr; + byte *buf = driver ? (byte*)cbin->driver.orig_bytes : + (byte*)cbin->nif.orig_bytes; + copy_binary_to_buffer(buf, 0, ((byte *) hb->data)+offset, 0, 8*size); + iov[0].iov_base = buf; + iov[0].iov_len = size; + binv[0] = cbin; + return 1; + } + } + } + return erts_ioq_iolist_to_vec(obj, iov, binv, cbin, bin_limit, driver); +} + +#endif + +#endif /* ERL_IO_QUEUE_H */ diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index f270d8baef..4cdef0200f 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -75,12 +75,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { * if only one lock use * the lock name)" */ -#ifdef ERTS_SMP { "driver_lock", "driver_name" }, { "port_lock", "port_id" }, -#endif { "port_data_lock", "address" }, -#ifdef ERTS_SMP { "bif_timers", NULL }, { "reg_tab", NULL }, { "proc_main", "pid" }, @@ -89,9 +86,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "hipe_mfait_lock", NULL }, #endif { "nodes_monitors", NULL }, -#ifdef ERTS_SMP { "resource_monitors", "address" }, -#endif { "driver_list", NULL }, { "proc_link", "pid" }, { "proc_msgq", "pid" }, @@ -104,7 +99,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "db_tab", "address" }, { "proc_status", "pid" }, { "proc_trace", "pid" }, - { "ports_snapshot", NULL }, { "db_tab_fix", "address" }, { "db_hash_slot", "address" }, { "node_table", NULL }, @@ -115,22 +109,18 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "fun_tab", NULL }, { "environ", NULL }, { "release_literal_areas", NULL }, -#endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, - { "pollset_rm_list", NULL }, { "removed_fd_pre_alloc_lock", "address" }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, { "migration_info_update", NULL }, { "run_queue", "address" }, -#ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, { "dirty_gc_info", NULL }, { "dirty_break_point_index", NULL }, -#endif { "process_table", NULL }, { "cpu_info", NULL }, { "pollset", "address" }, @@ -145,7 +135,6 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "async_enq_mtx", NULL }, { "msacc_list_mutex", NULL }, { "msacc_unmanaged_mutex", NULL }, -#ifdef ERTS_SMP { "atom_tab", NULL }, { "misc_op_list_pre_alloc_lock", "address" }, { "message_pre_alloc_lock", "address" }, @@ -156,28 +145,25 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "sys_msg_q", NULL }, { "tracer_mtx", NULL }, { "port_table", NULL }, -#endif { "magic_ref_table", "address" }, { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, + { "pollsets_lock", NULL }, { "alcu_allocator", "index" }, { "mseg", NULL }, -#ifdef ERTS_SMP { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, { "system_block", NULL }, - { "timeofday", NULL }, { "get_time", NULL }, { "get_corrected_time", NULL }, + { "runtime", NULL }, { "breakpoints", NULL }, - { "pollsets_lock", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, { "sched_stat", NULL }, -#endif { "async_init_mtx", NULL }, #ifdef __WIN32__ #ifdef DEBUG @@ -188,9 +174,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, -#ifdef ERTS_SMP { "os_monotonic_time", NULL }, -#endif { "erts_alloc_hard_debug", NULL }, { "hard_dbg_mseg", NULL }, { "erts_mmap", NULL } @@ -199,41 +183,20 @@ static erts_lc_lock_order_t erts_lock_order[] = { #define ERTS_LOCK_ORDER_SIZE \ (sizeof(erts_lock_order)/sizeof(erts_lc_lock_order_t)) -#define LOCK_IS_TYPE_ORDER_VIOLATION(LCK_FLG, LCKD_FLG) \ - (((LCKD_FLG) & (ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK)) \ - && ((LCK_FLG) \ - & ERTS_LC_FLG_LT_ALL \ - & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +#define LOCK_IS_TYPE_ORDER_VIOLATION(LCK_FLG, LCKD_FLG) \ + (((LCKD_FLG) & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_FLAGS_TYPE_SPINLOCK \ + && \ + ((LCK_FLG) & ERTS_LOCK_FLAGS_MASK_TYPE) != ERTS_LOCK_FLAGS_TYPE_SPINLOCK) static __decl_noreturn void __noreturn lc_abort(void); -static char * -lock_type(Uint16 flags) +static const char *rw_op_str(erts_lock_options_t options) { - switch (flags & ERTS_LC_FLG_LT_ALL) { - case ERTS_LC_FLG_LT_SPINLOCK: return "[spinlock]"; - case ERTS_LC_FLG_LT_RWSPINLOCK: return "[rw(spin)lock]"; - case ERTS_LC_FLG_LT_MUTEX: return "[mutex]"; - case ERTS_LC_FLG_LT_RWMUTEX: return "[rwmutex]"; - case ERTS_LC_FLG_LT_PROCLOCK: return "[proclock]"; - default: return ""; + if(options == ERTS_LOCK_OPTIONS_WRITE) { + ERTS_INTERNAL_ERROR("Only write flag present"); } -} -static char * -rw_op_str(Uint16 flags) -{ - switch (flags & ERTS_LC_FLG_LO_READ_WRITE) { - case ERTS_LC_FLG_LO_READ_WRITE: - return " (rw)"; - case ERTS_LC_FLG_LO_READ: - return " (r)"; - case ERTS_LC_FLG_LO_WRITE: - ERTS_INTERNAL_ERROR("Only write flag present"); - default: - break; - } - return ""; + return erts_lock_options_get_short_desc(options); } typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t; @@ -244,7 +207,8 @@ struct erts_lc_locked_lock_t_ { Sint16 id; char *file; unsigned int line; - Uint16 flags; + erts_lock_flags_t flags; + erts_lock_options_t taken_options; }; typedef struct { @@ -431,7 +395,7 @@ make_my_locked_locks(void) } static ERTS_INLINE erts_lc_locked_lock_t * -new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags, +new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc(); @@ -441,12 +405,13 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags, l_lck->extra = lck->extra; l_lck->file = file; l_lck->line = line; - l_lck->flags = lck->flags | op_flags; + l_lck->flags = lck->flags; + l_lck->taken_options = options; return l_lck; } static void -raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags, +raw_print_lock(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags, char* file, unsigned int line, char *suffix) { char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE @@ -458,16 +423,16 @@ raw_print_lock(char *prefix, Sint16 id, Wterm extra, Uint16 flags, erts_fprintf(stderr,"%p",_unchecked_boxed_val(extra)); else erts_fprintf(stderr,"%T",extra); - erts_fprintf(stderr,"%s",lock_type(flags)); + erts_fprintf(stderr,"[%s]",erts_lock_flags_get_type_name(flags)); if (file) erts_fprintf(stderr,"(%s:%d)",file,line); - erts_fprintf(stderr,"'%s%s",rw_op_str(flags),suffix); + erts_fprintf(stderr,"'(%s)%s",rw_op_str(flags),suffix); } static void -print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix) +print_lock2(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags, char *suffix) { raw_print_lock(prefix, id, extra, flags, NULL, 0, suffix); } @@ -522,9 +487,9 @@ uninitialized_lock(void) static void lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, - Uint16 op_flags) + erts_lock_options_t options) { - erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); + erts_fprintf(stderr, "%s (%s)", prefix, rw_op_str(options)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); lc_abort(); @@ -532,9 +497,9 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, static void unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, - Uint16 op_flags) + erts_lock_options_t options) { - erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); + erts_fprintf(stderr, "Unlocking (%s) ", rw_op_str(options)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); lc_abort(); @@ -745,84 +710,128 @@ erts_lc_get_lock_order_id(char *name) return (Sint16) -1; } +static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) +{ + if(locked_lock->id < comparand->id) { + return -1; + } else if(locked_lock->id > comparand->id) { + return 1; + } -static int -find_lock(erts_lc_locked_lock_t **l_lcks, erts_lc_lock_t *lck) + return 0; +} + +static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand) { - erts_lc_locked_lock_t *l_lck = *l_lcks; + int order = compare_locked_by_id(locked_lock, comparand); + + if(order) { + return order; + } else if(locked_lock->extra < comparand->extra) { + return -1; + } else if(locked_lock->extra > comparand->extra) { + return 1; + } - if (l_lck) { - if (l_lck->id == lck->id && l_lck->extra == lck->extra) { - if ((l_lck->flags & lck->flags) == lck->flags) - return 1; - return 0; - } - else if (l_lck->id < lck->id - || (l_lck->id == lck->id - && l_lck->extra < lck->extra)) { - for (l_lck = l_lck->next; l_lck; l_lck = l_lck->next) { - if (l_lck->id > lck->id - || (l_lck->id == lck->id - && l_lck->extra >= lck->extra)) { - *l_lcks = l_lck; - if (l_lck->id == lck->id - && l_lck->extra == lck->extra - && ((l_lck->flags & lck->flags) == lck->flags)) - return 1; - return 0; - } - } - } - else { - for (l_lck = l_lck->prev; l_lck; l_lck = l_lck->prev) { - if (l_lck->id < lck->id - || (l_lck->id == lck->id - && l_lck->extra <= lck->extra)) { - *l_lcks = l_lck; - if (l_lck->id == lck->id - && l_lck->extra == lck->extra - && ((l_lck->flags & lck->flags) == lck->flags)) - return 1; - return 0; - } - } - } + return 0; +} + +typedef int (*locked_compare_func)(erts_lc_locked_lock_t *, erts_lc_lock_t *); + +/* Searches through a list of taken locks, bailing when it hits an entry whose + * order relative to the search template is the opposite of the one at the + * start of the search. (*closest_neighbor) is either set to the exact match, + * or the one closest to it in the sort order. */ +static int search_locked_list(locked_compare_func compare, + erts_lc_locked_lock_t *locked_locks, + erts_lc_lock_t *search_template, + erts_lc_locked_lock_t **closest_neighbor) +{ + erts_lc_locked_lock_t *iterator = locked_locks; + + (*closest_neighbor) = iterator; + + if(iterator) { + int relative_order = compare(iterator, search_template); + + if(relative_order < 0) { + while((iterator = iterator->next) != NULL) { + relative_order = compare(iterator, search_template); + + if(relative_order >= 0) { + (*closest_neighbor) = iterator; + break; + } + } + } else if(relative_order > 0) { + while((iterator = iterator->prev) != NULL) { + relative_order = compare(iterator, search_template); + + if(relative_order <= 0) { + (*closest_neighbor) = iterator; + break; + } + } + } + + return relative_order == 0; } + return 0; } +/* Searches for a lock in the given list that matches search_template, and sets + * (*locked_locks) to the closest lock in the sort order. */ static int -find_id(erts_lc_locked_lock_t **l_lcks, Sint16 id) -{ - erts_lc_locked_lock_t *l_lck = *l_lcks; - - if (l_lck) { - if (l_lck->id == id) - return 1; - else if (l_lck->id < id) { - for (l_lck = l_lck->next; l_lck; l_lck = l_lck->next) { - if (l_lck->id >= id) { - *l_lcks = l_lck; - if (l_lck->id == id) - return 1; - return 0; - } - } - } - else { - for (l_lck = l_lck->prev; l_lck; l_lck = l_lck->prev) { - if (l_lck->id <= id) { - *l_lcks = l_lck; - if (l_lck->id == id) - return 1; - return 0; - } - } - } +find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template) +{ + erts_lc_locked_lock_t *closest_neighbor; + int found_lock; + + found_lock = search_locked_list(compare_locked_by_id_extra, + (*locked_locks), + search_template, + &closest_neighbor); + + (*locked_locks) = closest_neighbor; + + if(found_lock) { + erts_lock_options_t relevant_options; + erts_lock_flags_t relevant_flags; + + /* We only care about the options and flags that are set in the + * template. */ + relevant_options = (closest_neighbor->taken_options & search_template->taken_options); + relevant_flags = (closest_neighbor->flags & search_template->flags); + + return search_template->taken_options == relevant_options && + search_template->flags == relevant_flags; } + return 0; } +/* Searches for a lock in the given list by id, and sets (*locked_locks) to the + * closest lock in the sort order. */ +static int +find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id) +{ + erts_lc_locked_lock_t *closest_neighbor; + erts_lc_lock_t search_template; + int found_lock; + + search_template.id = id; + + found_lock = search_locked_list(compare_locked_by_id, + (*locked_locks), + &search_template, + &closest_neighbor); + + (*locked_locks) = closest_neighbor; + + return found_lock; +} + void erts_lc_have_locks(int *resv, erts_lc_lock_t *locks, int len) { @@ -918,17 +927,17 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } void -erts_lc_check_no_locked_of_type(Uint16 flags) +erts_lc_check_no_locked_of_type(erts_lock_flags_t type) { erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); if (l_lcks) { erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { - if (l_lck->flags & flags) { + if ((l_lck->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) { erts_fprintf(stderr, "Locked lock of type %s found which isn't " "allowed here!\n", - lock_type(l_lck->flags)); + erts_lock_flags_get_type_name(l_lck->flags)); print_curr_locks(l_lcks); lc_abort(); } @@ -937,7 +946,7 @@ erts_lc_check_no_locked_of_type(Uint16 flags) } int -erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) +erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { #ifdef ERTS_LC_DO_NOT_FORCE_BUSY_TRYLOCK_ON_LOCK_ORDER_VIOLATION return 0; @@ -986,7 +995,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) if (tl_lck->id < lck->id || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { if (tl_lck->id == lck->id && tl_lck->extra == lck->extra) - lock_twice("Trylocking", l_lcks, lck, op_flags); + lock_twice("Trylocking", l_lcks, lck, options); break; } } @@ -1008,7 +1017,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) #endif } -void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks; @@ -1021,7 +1030,7 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, return; l_lcks = make_my_locked_locks(); - l_lck = locked ? new_locked_lock(lck, op_flags, file, line) : NULL; + l_lck = locked ? new_locked_lock(lck, options, file, line) : NULL; if (!l_lcks->locked.last) { ASSERT(!l_lcks->locked.first); @@ -1039,7 +1048,7 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, if (tl_lck->id < lck->id || (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) { if (tl_lck->id == lck->id && tl_lck->extra == lck->extra) - lock_twice("Trylocking", l_lcks, lck, op_flags); + lock_twice("Trylocking", l_lcks, lck, options); if (locked) { l_lck->next = tl_lck->next; l_lck->prev = tl_lck; @@ -1062,14 +1071,14 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, } -void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks = make_my_locked_locks(); erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; if (!find_lock(&l_lck, lck)) required_not_locked(l_lcks, lck); - l_lck = new_locked_lock(lck, op_flags, file, line); + l_lck = new_locked_lock(lck, options, file, line); if (!l_lcks->required.last) { ASSERT(!l_lcks->required.first); l_lck->next = l_lck->prev = NULL; @@ -1109,7 +1118,7 @@ void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags, } } -void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { erts_lc_locked_locks_t *l_lcks = make_my_locked_locks(); erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; @@ -1137,7 +1146,7 @@ void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags) lc_free((void *) l_lck); } -void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line) { erts_lc_locked_locks_t *l_lcks; @@ -1150,7 +1159,7 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, return; l_lcks = make_my_locked_locks(); - l_lck = new_locked_lock(lck, op_flags, file, line); + l_lck = new_locked_lock(lck, options, file, line); if (!l_lcks->locked.last) { ASSERT(!l_lcks->locked.first); @@ -1166,12 +1175,12 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, l_lcks->locked.last = l_lck; } else if (l_lcks->locked.last->id == lck->id && l_lcks->locked.last->extra == lck->extra) - lock_twice("Locking", l_lcks, lck, op_flags); + lock_twice("Locking", l_lcks, lck, options); else lock_order_violation(l_lcks, lck); } -void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { erts_lc_locked_locks_t *l_lcks; erts_lc_locked_lock_t *l_lck; @@ -1192,8 +1201,8 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags) for (l_lck = l_lcks ? l_lcks->locked.last : NULL; l_lck; l_lck = l_lck->prev) { if (l_lck->id == lck->id && l_lck->extra == lck->extra) { - if ((l_lck->flags & ERTS_LC_FLG_LO_ALL) != op_flags) - unlock_op_mismatch(l_lcks, lck, op_flags); + if ((l_lck->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options) + unlock_op_mismatch(l_lcks, lck, options); if (l_lck->prev) l_lck->prev->next = l_lck->next; else @@ -1210,7 +1219,7 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags) unlock_of_not_locked(l_lcks, lck); } -void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags) +void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options) { erts_lc_locked_locks_t *l_lcks; erts_lc_locked_lock_t *l_lck; @@ -1274,23 +1283,25 @@ void erts_lc_unrequire_lock(erts_lc_lock_t *lck) } void -erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) +erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags) { lck->id = erts_lc_get_lock_order_id(name); lck->extra = (UWord) &lck->extra; ASSERT(is_not_immed(lck->extra)); lck->flags = flags; + lck->taken_options = 0; lck->inited = ERTS_LC_INITITALIZED; } void -erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra) +erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags, Eterm extra) { lck->id = erts_lc_get_lock_order_id(name); lck->extra = extra; ASSERT(is_immed(lck->extra)); lck->flags = flags; + lck->taken_options = 0; lck->inited = ERTS_LC_INITITALIZED; } @@ -1304,6 +1315,7 @@ erts_lc_destroy_lock(erts_lc_lock_t *lck) lck->id = -1; lck->extra = THE_NON_VALUE; lck->flags = 0; + lck->taken_options = 0; } void diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 18296d1fec..5c2c38e8f2 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -36,6 +36,8 @@ #ifdef ERTS_ENABLE_LOCK_CHECK +#include "erl_lock_flags.h" + #ifndef ERTS_ENABLE_LOCK_POSITION /* Enable in order for _x variants of mtx functions to be used. */ #define ERTS_ENABLE_LOCK_POSITION 1 @@ -44,36 +46,14 @@ typedef struct { int inited; Sint16 id; - Uint16 flags; + erts_lock_flags_t flags; + erts_lock_options_t taken_options; UWord extra; } erts_lc_lock_t; #define ERTS_LC_INITITALIZED 0x7f7f7f7f - -#define ERTS_LC_FLG_LT_SPINLOCK (((Uint16) 1) << 0) -#define ERTS_LC_FLG_LT_RWSPINLOCK (((Uint16) 1) << 1) -#define ERTS_LC_FLG_LT_MUTEX (((Uint16) 1) << 2) -#define ERTS_LC_FLG_LT_RWMUTEX (((Uint16) 1) << 3) -#define ERTS_LC_FLG_LT_PROCLOCK (((Uint16) 1) << 4) - -#define ERTS_LC_FLG_LO_READ (((Uint16) 1) << 5) -#define ERTS_LC_FLG_LO_WRITE (((Uint16) 1) << 6) - -#define ERTS_LC_FLG_LO_READ_WRITE (ERTS_LC_FLG_LO_READ \ - | ERTS_LC_FLG_LO_WRITE) - -#define ERTS_LC_FLG_LT_ALL (ERTS_LC_FLG_LT_SPINLOCK \ - | ERTS_LC_FLG_LT_RWSPINLOCK \ - | ERTS_LC_FLG_LT_MUTEX \ - | ERTS_LC_FLG_LT_RWMUTEX \ - | ERTS_LC_FLG_LT_PROCLOCK) - -#define ERTS_LC_FLG_LO_ALL (ERTS_LC_FLG_LO_READ \ - | ERTS_LC_FLG_LO_WRITE) - - -#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), (X)} +#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), 0, (X)} void erts_lc_init(void); void erts_lc_late_init(void); @@ -83,31 +63,31 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); -void erts_lc_check_no_locked_of_type(Uint16 flags); -int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); -void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_check_no_locked_of_type(erts_lock_flags_t flags); +int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options); +void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line); -void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line); -void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags); -void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags); +void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options); +void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options); int erts_lc_trylock_force_busy(erts_lc_lock_t *lck); void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck, char* file, unsigned int line); void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line); void erts_lc_unlock(erts_lc_lock_t *lck); void erts_lc_might_unlock(erts_lc_lock_t *lck); -void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags); -void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra); +void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags); +void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, erts_lock_flags_t flags, Eterm extra); void erts_lc_destroy_lock(erts_lc_lock_t *lck); void erts_lc_fail(char *fmt, ...); int erts_lc_assert_failed(char *file, int line, char *assertion); void erts_lc_set_thread_name(char *thread_name); void erts_lc_pll(void); -void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags, +void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options, char *file, unsigned int line); -void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); +void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options); void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line); void erts_lc_unrequire_lock(erts_lc_lock_t *lck); @@ -116,13 +96,7 @@ int erts_lc_is_emu_thr(void); #define ERTS_LC_ASSERT(A) \ ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A))) -#ifdef ERTS_SMP -#define ERTS_SMP_LC_ASSERT(A) ERTS_LC_ASSERT(A) -#else -#define ERTS_SMP_LC_ASSERT(A) ((void) 1) -#endif #else /* #ifdef ERTS_ENABLE_LOCK_CHECK */ -#define ERTS_SMP_LC_ASSERT(A) ((void) 1) #define ERTS_LC_ASSERT(A) ((void) 1) #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */ diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 678bc43f04..1ae6076b12 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -18,51 +18,37 @@ * %CopyrightEnd% */ -/* - * Description: Statistics for locks. - * - * Author: Björn-Egil Dahlberg - * Date: 2008-07-03 - */ - #ifdef HAVE_CONFIG_H # include "config.h" #endif -/* Needed for VxWorks va_arg */ +#ifdef ERTS_ENABLE_LOCK_COUNT + #include "sys.h" -#ifdef ERTS_ENABLE_LOCK_COUNT +#include "global.h" #include "erl_lock_count.h" -#include "ethread.h" -#include "erl_term.h" -#include "atom.h" -#include <stdio.h> - -/* globals, dont access these without locks or blocks */ +#include "erl_thr_progress.h" -ethr_mutex lcnt_data_lock; -erts_lcnt_data_t *erts_lcnt_data; -Uint16 erts_lcnt_rt_options; -erts_lcnt_time_t timer_start; -const char *str_undefined = "undefined"; +#include "erl_node_tables.h" +#include "erl_alloc_util.h" +#include "erl_check_io.h" +#include "erl_poll.h" +#include "erl_db.h" -static ethr_tsd_key lcnt_thr_data_key; -static int lcnt_n_thr; -static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; +#define LCNT_MAX_CARRIER_ENTRIES 255 -/* local functions */ +/* - Locals that are shared with the header implementation - */ -static ERTS_INLINE void lcnt_lock(void) { - ethr_mutex_lock(&lcnt_data_lock); -} +#ifdef DEBUG +int lcnt_initialization_completed__; +#endif -static ERTS_INLINE void lcnt_unlock(void) { - ethr_mutex_unlock(&lcnt_data_lock); -} +erts_lock_flags_t lcnt_category_mask__; +ethr_tsd_key lcnt_thr_data_key__; -const int log2_tab64[64] = { +const int lcnt_log2_tab64__[64] = { 63, 0, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, 33, 42, 3, 61, 51, 37, 40, 49, 18, 28, 20, @@ -72,635 +58,602 @@ const int log2_tab64[64] = { 56, 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5}; -static ERTS_INLINE int lcnt_log2(Uint64 v) { - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - v |= v >> 32; - return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; -} - -static char* lcnt_lock_type(Uint16 flag) { - switch(flag & ERTS_LCNT_LT_ALL) { - case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; - case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; - case ERTS_LCNT_LT_MUTEX: return "mutex"; - case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; - case ERTS_LCNT_LT_PROCLOCK: return "proclock"; - default: return ""; - } -} +/* - Local variables - */ -static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) { - ethr_atomic_set(&stats->tries, 0); - ethr_atomic_set(&stats->colls, 0); - stats->timer.s = 0; - stats->timer.ns = 0; - stats->timer_n = 0; - stats->file = (char *)str_undefined; - stats->line = 0; - sys_memzero(stats->hist.ns, sizeof(stats->hist.ns)); -} +typedef struct lcnt_static_lock_ref_ { + erts_lcnt_ref_t *reference; -static void lcnt_time(erts_lcnt_time_t *time) { - /* - * erts_sys_hrtime() is the highest resolution - * we could find, it may or may not be monotonic... - */ - ErtsMonotonicTime mtime = erts_sys_hrtime(); - time->s = (unsigned long) (mtime / 1000000000LL); - time->ns = (unsigned long) (mtime - 1000000000LL*time->s); -} + erts_lock_flags_t flags; + const char *name; + Eterm id; -static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { - long ds; - long dns; + struct lcnt_static_lock_ref_ *next; +} lcnt_static_lock_ref_t; - ds = t1->s - t0->s; - dns = t1->ns - t0->ns; +static ethr_atomic_t lcnt_static_lock_registry; - /* the difference should not be able to get bigger than 1 sec in ns*/ +static erts_lcnt_lock_info_list_t lcnt_current_lock_list; +static erts_lcnt_lock_info_list_t lcnt_deleted_lock_list; - if (dns < 0) { - ds -= 1; - dns += 1000000000LL; - } +static erts_lcnt_time_t lcnt_timer_start; - ASSERT(ds >= 0); +static int lcnt_preserve_info; - d->s = ds; - d->ns = dns; -} +/* local functions */ + +static void lcnt_clear_stats(erts_lcnt_lock_info_t *info) { + size_t i; + + for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[i]; -/* difference d must be non-negative */ + sys_memzero(&stats->wait_time_histogram, sizeof(stats->wait_time_histogram)); -static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { - t->s += d->s; - t->ns += d->ns; + stats->total_time_waited.s = 0; + stats->total_time_waited.ns = 0; - t->s += t->ns / 1000000000LL; - t->ns = t->ns % 1000000000LL; + stats->times_waited = 0; + + stats->file = NULL; + stats->line = 0; + + ethr_atomic_set(&stats->attempts, 0); + ethr_atomic_set(&stats->collisions, 0); + } + + info->location_count = 1; } -static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { - erts_lcnt_thread_data_t *eltd; +static lcnt_thread_data_t__ *lcnt_thread_data_alloc(void) { + lcnt_thread_data_t__ *eltd = + (lcnt_thread_data_t__*)malloc(sizeof(lcnt_thread_data_t__)); - eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t)); - if (!eltd) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + if(!eltd) { + ERTS_INTERNAL_ERROR("Failed to allocate lcnt thread data."); } + eltd->timer_set = 0; eltd->lock_in_conflict = 0; - eltd->id = lcnt_n_thr++; - /* set thread data to array */ - lcnt_thread_data[eltd->id] = eltd; - return eltd; } -static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { - return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); -} +/* - List operations - + * + * Info entries are kept in a doubly linked list where each entry is locked + * with its neighbors rather than a global lock. Deletion is rather quick, but + * insertion is still serial since the head becomes a de facto global lock. + * + * We rely on ad-hoc spinlocks to avoid "recursing" into this module. */ -/* debug */ +#define LCNT_SPINLOCK_YIELD_ITERATIONS 50 -#if 0 -static char* lock_opt(Uint16 flag) { - if ((flag & ERTS_LCNT_LO_WRITE) && (flag & ERTS_LCNT_LO_READ)) return "rw"; - if (flag & ERTS_LCNT_LO_READ ) return "r "; - if (flag & ERTS_LCNT_LO_WRITE) return " w"; - return "--"; -} +#define LCNT_SPINLOCK_HELPER_INIT \ + Uint failed_spin_count = 0; -static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) { - erts_aint_t w_state, r_state; - char *type; +#define LCNT_SPINLOCK_HELPER_YIELD \ + do { \ + failed_spin_count++; \ + if(!(failed_spin_count % LCNT_SPINLOCK_YIELD_ITERATIONS)) { \ + erts_thr_yield(); \ + } else { \ + ERTS_SPIN_BODY; \ + } \ + } while(0) - if (strcmp(lock->name, "run_queue") != 0) return; - type = lcnt_lock_type(lock->flag); - r_state = ethr_atomic_read(&lock->r_state); - w_state = ethr_atomic_read(&lock->w_state); +static void lcnt_unlock_list_entry(erts_lcnt_lock_info_t *info) { + ethr_atomic32_set_relb(&info->lock, 0); +} - if (lock->flag & flag) { - erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n", - action, - lock->name, - r_state, - w_state, - type, - lock->id); - } +static int lcnt_try_lock_list_entry(erts_lcnt_lock_info_t *info) { + return ethr_atomic32_cmpxchg_acqb(&info->lock, 1, 0) == 0; } -#endif -static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) { - unsigned int i; - erts_lcnt_lock_stats_t *stats = NULL; +static void lcnt_lock_list_entry(erts_lcnt_lock_info_t *info) { + LCNT_SPINLOCK_HELPER_INIT; - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { - for (i = 0; i < lock->n_stats; i++) { - if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { - return &(lock->stats[i]); - } - } - if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { - stats = &lock->stats[lock->n_stats]; - lock->n_stats++; - stats->file = file; - stats->line = line; - return stats; - } + while(!lcnt_try_lock_list_entry(info)) { + LCNT_SPINLOCK_HELPER_YIELD; } - return &lock->stats[0]; } -static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) { - int idx; - unsigned long r; +static void lcnt_lock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) { + LCNT_SPINLOCK_HELPER_INIT; - if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { - idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; - } else { - r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; - if (r) idx = lcnt_log2(r); - else idx = 0; + for(;;) { + if(!lcnt_try_lock_list_entry(info)) + goto retry_after_entry_failed; + if(!lcnt_try_lock_list_entry(info->next)) + goto retry_after_next_failed; + if(!lcnt_try_lock_list_entry(info->prev)) + goto retry_after_prev_failed; + + return; + + retry_after_prev_failed: + lcnt_unlock_list_entry(info->next); + retry_after_next_failed: + lcnt_unlock_list_entry(info); + retry_after_entry_failed: + LCNT_SPINLOCK_HELPER_YIELD; } - hist->ns[idx]++; } -static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, - erts_lcnt_time_t *time_wait) { +static void lcnt_unlock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) { + lcnt_unlock_list_entry(info->prev); + lcnt_unlock_list_entry(info->next); + lcnt_unlock_list_entry(info); +} - ethr_atomic_inc(&stats->tries); +static void lcnt_insert_list_entry(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t *info) { + erts_lcnt_lock_info_t *next, *prev; - if (lock_in_conflict) - ethr_atomic_inc(&stats->colls); + prev = &list->head; - if (time_wait) { - lcnt_time_add(&(stats->timer), time_wait); - stats->timer_n++; - lcnt_update_stats_hist(&stats->hist,time_wait); - } -} + lcnt_lock_list_entry(prev); -/* interface */ + next = prev->next; -void erts_lcnt_init() { - erts_lcnt_thread_data_t *eltd = NULL; + lcnt_lock_list_entry(next); - /* init lock */ - if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); + info->next = next; + info->prev = prev; - /* init tsd */ - lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data"); + prev->next = info; + next->prev = info; - lcnt_lock(); + lcnt_unlock_list_entry(next); + lcnt_unlock_list_entry(prev); +} + +static void lcnt_insert_list_carrier(erts_lcnt_lock_info_list_t *list, + erts_lcnt_lock_info_carrier_t *carrier) { + erts_lcnt_lock_info_t *next, *prev; + size_t i; - erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK; - eltd = lcnt_thread_data_alloc(); - ethr_tsd_set(lcnt_thr_data_key, eltd); + for(i = 0; i < carrier->entry_count; i++) { + erts_lcnt_lock_info_t *info = &carrier->entries[i]; - /* init lcnt structure */ - erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t)); - if (!erts_lcnt_data) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + info->prev = &carrier->entries[i - 1]; + info->next = &carrier->entries[i + 1]; } - erts_lcnt_data->current_locks = erts_lcnt_list_init(); - erts_lcnt_data->deleted_locks = erts_lcnt_list_init(); - lcnt_unlock(); + prev = &list->head; + + lcnt_lock_list_entry(prev); + + next = prev->next; + + lcnt_lock_list_entry(next); + + next->prev = &carrier->entries[carrier->entry_count - 1]; + carrier->entries[carrier->entry_count - 1].next = next; + prev->next = &carrier->entries[0]; + carrier->entries[0].prev = prev; + + lcnt_unlock_list_entry(next); + lcnt_unlock_list_entry(prev); } -void erts_lcnt_late_init() { - /* set start timer and zero statistics */ - erts_lcnt_clear_counters(); - erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); +static void lcnt_init_list(erts_lcnt_lock_info_list_t *list) { + /* Ensure that ref_count operations explode when touching the sentinels in + * DEBUG mode. */ + ethr_atomic_init(&(list->head.ref_count), -1); + ethr_atomic_init(&(list->tail.ref_count), -1); + + ethr_atomic32_init(&(list->head.lock), 0); + (list->head).next = &list->tail; + (list->head).prev = &list->tail; + + ethr_atomic32_init(&(list->tail.lock), 0); + (list->tail).next = &list->head; + (list->tail).prev = &list->head; } -/* list operations */ +/* - Carrier operations - */ -/* BEGIN ASSUMPTION: lcnt_data_lock taken */ +int lcnt_thr_progress_unmanaged_delay__(void) { + return erts_thr_progress_unmanaged_delay(); +} -erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { - erts_lcnt_lock_list_t *list; +void lcnt_thr_progress_unmanaged_continue__(int handle) { + return erts_thr_progress_unmanaged_continue(handle); +} - list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t)); - if (!list) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); - } - list->head = NULL; - list->tail = NULL; - list->n = 0; - return list; +void lcnt_deallocate_carrier__(erts_lcnt_lock_info_carrier_t *carrier) { + ASSERT(ethr_atomic_read(&carrier->ref_count) == 0); + erts_free(ERTS_ALC_T_LCNT_CARRIER, (void*)carrier); } -static void lcnt_list_free(erts_lcnt_lock_t *head) { - erts_lcnt_lock_t *lock, *next; +static void lcnt_thr_prg_cleanup_carrier(void *data) { + erts_lcnt_lock_info_carrier_t *carrier = data; + size_t entry_count, i; + + /* carrier->entry_count will be replaced with garbage if it's deallocated + * on the final iteration, so we'll tuck it away to get a clean exit. */ + entry_count = carrier->entry_count; - lock = head; + for(i = 0; i < entry_count; i++) { + ASSERT(ethr_atomic_read(&carrier->ref_count) >= (entry_count - i)); - while(lock != NULL) { - next = lock->next; - free(lock); - lock = next; + erts_lcnt_release_lock_info(&carrier->entries[i]); } } -void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - erts_lcnt_lock_t *tail = NULL; +static void lcnt_schedule_carrier_cleanup(void *data) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + + /* We can't issue cleanup jobs on anything other than normal schedulers, so + * we move to the first scheduler if required. */ - tail = list->tail; - if (tail) { - tail->next = lock; - lock->prev = tail; + if(!esdp || esdp->type != ERTS_SCHED_NORMAL) { + erts_schedule_misc_aux_work(1, &lcnt_schedule_carrier_cleanup, data); } else { - list->head = lock; - lock->prev = NULL; - ASSERT(!lock->next); + erts_lcnt_lock_info_carrier_t *carrier = data; + size_t carrier_size; + + carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) + + sizeof(erts_lcnt_lock_info_t) * carrier->entry_count; + + erts_schedule_thr_prgr_later_cleanup_op(&lcnt_thr_prg_cleanup_carrier, + data, (ErtsThrPrgrLaterOp*)&carrier->release_entries, carrier_size); } - lock->next = NULL; - list->tail = lock; +} - list->n++; +static void lcnt_info_deallocate(erts_lcnt_lock_info_t *info) { + lcnt_release_carrier__(info->carrier); } -void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - if (lock->next) lock->next->prev = lock->prev; - if (lock->prev) lock->prev->next = lock->next; - if (list->head == lock) list->head = lock->next; - if (list->tail == lock) list->tail = lock->prev; +static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) { + ASSERT(ethr_atomic_read(&info->ref_count) == 0); + + if(lcnt_preserve_info) { + ethr_atomic_set(&info->ref_count, 1); + + /* Move straight to deallocation the next time around. */ + info->dispose = &lcnt_info_deallocate; - lock->prev = NULL; - lock->next = NULL; - list->n--; + lcnt_insert_list_entry(&lcnt_deleted_lock_list, info); + } else { + lcnt_info_deallocate(info); + } } -/* END ASSUMPTION: lcnt_data_lock taken */ +static void lcnt_lock_info_init_helper(erts_lcnt_lock_info_t *info) { + ethr_atomic_init(&info->ref_count, 1); + ethr_atomic32_init(&info->lock, 0); + + ethr_atomic_init(&info->r_state, 0); + ethr_atomic_init(&info->w_state, 0); -/* lock operations */ + info->dispose = &lcnt_info_dispose; -/* interface to erl_threads.h */ -/* only lock on init and destroy, all others should use atomics */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { - erts_lcnt_init_lock_x(lock, name, flag, NIL); + lcnt_clear_stats(info); } -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { - int i; +erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_count) { + erts_lcnt_lock_info_carrier_t *result; + size_t carrier_size, i; - if (flag & ERTS_LCNT_LT_DISABLE) { - ERTS_LCNT_CLEAR_FLAG(lock); - return; - } + ASSERT(entry_count > 0 && entry_count <= LCNT_MAX_CARRIER_ENTRIES); + ASSERT(lcnt_initialization_completed__); - lock->next = NULL; - lock->prev = NULL; - lock->flag = flag; - lock->name = name; - lock->id = id; + carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) + + sizeof(erts_lcnt_lock_info_t) * entry_count; - ethr_atomic_init(&lock->r_state, 0); - ethr_atomic_init(&lock->w_state, 0); -#ifdef DEBUG - ethr_atomic_init(&lock->flowstate, 0); -#endif + result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size); + result->entry_count = entry_count; - lock->n_stats = 1; + ethr_atomic_init(&result->ref_count, entry_count); - for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - lcnt_clear_stats(&lock->stats[i]); - } + for(i = 0; i < entry_count; i++) { + erts_lcnt_lock_info_t *info = &result->entries[i]; - lcnt_lock(); - erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); - lcnt_unlock(); -} + lcnt_lock_info_init_helper(info); -/* init empty, instead of zero struct - * used by process locks probes - */ -void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { - lock->next = NULL; - lock->prev = NULL; - lock->flag = 0; - lock->name = NULL; - lock->id = NIL; - ethr_atomic_init(&lock->r_state, 0); - ethr_atomic_init(&lock->w_state, 0); -#ifdef DEBUG - ethr_atomic_init(&lock->flowstate, 0); -#endif - lock->n_stats = 0; - sys_memzero(lock->stats, sizeof(lock->stats)); -} -/* destroy lock */ -void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - lcnt_lock(); - - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { - erts_lcnt_lock_t *deleted_lock; - /* copy structure and insert the copy */ - deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); - if (!deleted_lock) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); - } - memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); - deleted_lock->next = NULL; - deleted_lock->prev = NULL; - erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); + info->carrier = result; } - /* delete original */ - erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock); - ERTS_LCNT_CLEAR_FLAG(lock); - lcnt_unlock(); + return result; } -/* lock */ +void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier) { + ethr_sint_t swapped_carrier; -void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - erts_aint_t r_state = 0, w_state = 0; - erts_lcnt_thread_data_t *eltd; +#ifdef DEBUG + int i; - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + /* Verify that all locks share the same categories/static property; all + * other flags are fair game. */ + for(i = 1; i < carrier->entry_count; i++) { + const erts_lock_flags_t SIGNIFICANT_DIFF_MASK = + ERTS_LOCK_FLAGS_MASK_CATEGORY | ERTS_LOCK_FLAGS_PROPERTY_STATIC; - eltd = lcnt_get_thread_data(); - ASSERT(eltd); + erts_lcnt_lock_info_t *previous, *current; - w_state = ethr_atomic_read(&lock->w_state); + previous = &carrier->entries[i - 1]; + current = &carrier->entries[i]; - if (option & ERTS_LCNT_LO_WRITE) { - r_state = ethr_atomic_read(&lock->r_state); - ethr_atomic_inc( &lock->w_state); - } - if (option & ERTS_LCNT_LO_READ) { - ethr_atomic_inc( &lock->r_state); + ASSERT(!((previous->flags ^ current->flags) & SIGNIFICANT_DIFF_MASK)); } +#endif - /* we cannot acquire w_lock if either w or r are taken */ - /* we cannot acquire r_lock if w_lock is taken */ + swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)carrier, (ethr_sint_t)NULL); - if ((w_state > 0) || (r_state > 0)) { - eltd->lock_in_conflict = 1; - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; + if(swapped_carrier != (ethr_sint_t)NULL) { +#ifdef DEBUG + ASSERT(ethr_atomic_read(&carrier->ref_count) == carrier->entry_count); + ethr_atomic_set(&carrier->ref_count, 0); +#endif + + lcnt_deallocate_carrier__(carrier); } else { - eltd->lock_in_conflict = 0; + lcnt_insert_list_carrier(&lcnt_current_lock_list, carrier); } } -void erts_lcnt_lock(erts_lcnt_lock_t *lock) { - erts_aint_t w_state; - erts_lcnt_thread_data_t *eltd; +void erts_lcnt_uninstall(erts_lcnt_ref_t *ref) { + ethr_sint_t previous_carrier, swapped_carrier; - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + previous_carrier = ethr_atomic_read(ref); + swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)NULL, previous_carrier); - w_state = ethr_atomic_read(&lock->w_state); - ethr_atomic_inc(&lock->w_state); - eltd = lcnt_get_thread_data(); + if(previous_carrier && previous_carrier == swapped_carrier) { + lcnt_schedule_carrier_cleanup((void*)previous_carrier); + } +} - ASSERT(eltd); +/* - Static lock registry - + * + * Since static locks can be trusted to never disappear, we can track them + * pretty cheaply and won't need to bother writing an "erts_lcnt_update_xx" + * variant. */ - if (w_state > 0) { - eltd->lock_in_conflict = 1; - /* only set the timer if nobody else has it - * This should only happen when proc_locks aquires several locks - * 'atomicly'. All other locks will block the thread if w_state > 0 - * i.e. locked. - */ - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; - } else { - eltd->lock_in_conflict = 0; - } +static void lcnt_init_static_lock_registry(void) { + ethr_atomic_init(&lcnt_static_lock_registry, (ethr_sint_t)NULL); } -/* if a lock wasn't really a lock operation, bad bad process locks */ +static void lcnt_update_static_locks(void) { + lcnt_static_lock_ref_t *iterator = + (lcnt_static_lock_ref_t*)ethr_atomic_read(&lcnt_static_lock_registry); -void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) { - /* should check if this thread was "waiting" */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + while(iterator != NULL) { + if(!erts_lcnt_check_enabled(iterator->flags)) { + erts_lcnt_uninstall(iterator->reference); + } else if(!erts_lcnt_check_ref_installed(iterator->reference)) { + erts_lcnt_lock_info_carrier_t *carrier = erts_lcnt_create_lock_info_carrier(1); - ethr_atomic_dec(&lock->w_state); -} + erts_lcnt_init_lock_info_idx(carrier, 0, iterator->name, iterator->id, iterator->flags); -/* - * erts_lcnt_lock_post - * - * Used when we get a lock (i.e. directly after a lock operation) - * if the timer was set then we had to wait for the lock - * lock_post will calculate the wait time. - */ + erts_lcnt_install(iterator->reference, carrier); + } -void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) { - erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0); + iterator = iterator->next; + } } -void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line) { - erts_lcnt_thread_data_t *eltd; - erts_lcnt_time_t timer; - erts_lcnt_time_t time_wait; - erts_lcnt_lock_stats_t *stats; -#ifdef DEBUG - erts_aint_t flowstate; -#endif +void lcnt_register_static_lock__(erts_lcnt_ref_t *reference, const char *name, Eterm id, + erts_lock_flags_t flags) { + lcnt_static_lock_ref_t *lock = malloc(sizeof(lcnt_static_lock_ref_t)); + int retry_insertion; - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + ASSERT(flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC); -#ifdef DEBUG - if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc(&lock->flowstate); - } -#endif + lock->reference = reference; + lock->flags = flags; + lock->name = name; + lock->id = id; - eltd = lcnt_get_thread_data(); + do { + ethr_sint_t swapped_head; - ASSERT(eltd); + lock->next = (lcnt_static_lock_ref_t*)ethr_atomic_read(&lcnt_static_lock_registry); - /* if lock was in conflict, time it */ - stats = lcnt_get_lock_stats(lock, file, line); - if (eltd->timer_set) { - lcnt_time(&timer); + swapped_head = ethr_atomic_cmpxchg_acqb( + &lcnt_static_lock_registry, + (ethr_sint_t)lock, + (ethr_sint_t)lock->next); - lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); - lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); - eltd->timer_set--; - ASSERT(eltd->timer_set >= 0); - } else { - lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); - } + retry_insertion = (swapped_head != (ethr_sint_t)lock->next); + } while(retry_insertion); +} + +/* - Initialization - */ + +void erts_lcnt_pre_thr_init() { + /* Ensure that the dependency hack mentioned in the header doesn't + * explode at runtime. */ + ERTS_CT_ASSERT(sizeof(LcntThrPrgrLaterOp) >= sizeof(ErtsThrPrgrLaterOp)); + ERTS_CT_ASSERT(ERTS_THR_PRGR_DHANDLE_MANAGED == + (ErtsThrPrgrDelayHandle)LCNT_THR_PRGR_DHANDLE_MANAGED); + lcnt_init_list(&lcnt_current_lock_list); + lcnt_init_list(&lcnt_deleted_lock_list); + + lcnt_init_static_lock_registry(); } -/* unlock */ +void erts_lcnt_post_thr_init() { + /* ASSUMPTION: this is safe since it runs prior to the creation of other + * threads (Directly after ethread init). */ + + ethr_tsd_key_create(&lcnt_thr_data_key__, "lcnt_data"); -void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state); + erts_lcnt_thread_setup(); } -void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; +void erts_lcnt_late_init() { + /* Set start timer and zero all statistics */ + erts_lcnt_clear_counters(); + erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); + #ifdef DEBUG - { - erts_aint_t w_state; - erts_aint_t flowstate; - - /* flowstate */ - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 1); - ethr_atomic_dec(&lock->flowstate); - - /* write state */ - w_state = ethr_atomic_read(&lock->w_state); - ASSERT(w_state > 0); - } + /* It's safe to use erts_alloc and thread progress past this point. */ + lcnt_initialization_completed__ = 1; #endif - ethr_atomic_dec(&lock->w_state); } -/* trylock */ +void erts_lcnt_post_startup(void) { + /* Default to capturing everything to match the behavior of the old lock + * counter build. */ + erts_lcnt_set_category_mask(ERTS_LOCK_FLAGS_MASK_CATEGORY); +} -void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - /* Determine lock_state via res instead of state */ - if (res != EBUSY) { - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); - } else { - ethr_atomic_inc(&lock->stats[0].tries); - ethr_atomic_inc(&lock->stats[0].colls); +void erts_lcnt_thread_setup() { + lcnt_thread_data_t__ *eltd = lcnt_thread_data_alloc(); + + ASSERT(eltd); + + ethr_tsd_set(lcnt_thr_data_key__, eltd); +} + +void erts_lcnt_thread_exit_handler() { + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); + + if (eltd) { + free(eltd); } } +/* - BIF interface - */ -void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { - /* Determine lock_state via res instead of state */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - if (res != EBUSY) { +void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info) { #ifdef DEBUG - { - erts_aint_t flowstate; - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); - } + ASSERT(ethr_atomic_inc_read_acqb(&info->ref_count) >= 2); +#else + ethr_atomic_inc_acqb(&info->ref_count); #endif - ethr_atomic_inc(&lock->w_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); +} + +void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info) { + ethr_sint_t count; + + /* We need to acquire the lock before decrementing ref_count to avoid + * racing with list iteration; there's a short window between reading the + * reference to info and increasing its ref_count. */ + lcnt_lock_list_entry_with_neighbors(info); + + count = ethr_atomic_dec_read(&info->ref_count); + + ASSERT(count >= 0); + + if(count > 0) { + lcnt_unlock_list_entry_with_neighbors(info); } else { - ethr_atomic_inc(&lock->stats[0].tries); - ethr_atomic_inc(&lock->stats[0].colls); + (info->next)->prev = info->prev; + (info->prev)->next = info->next; + + lcnt_unlock_list_entry_with_neighbors(info); + + info->dispose(info); } } -/* thread operations */ +erts_lock_flags_t erts_lcnt_get_category_mask() { + return lcnt_category_mask__; +} -void erts_lcnt_thread_setup(void) { - erts_lcnt_thread_data_t *eltd; +void erts_lcnt_set_category_mask(erts_lock_flags_t mask) { + erts_lock_flags_t changed_categories; - lcnt_lock(); - /* lock for thread id global update */ - eltd = lcnt_thread_data_alloc(); - lcnt_unlock(); - ASSERT(eltd); - ethr_tsd_set(lcnt_thr_data_key, eltd); -} + ASSERT(!(mask & ~ERTS_LOCK_FLAGS_MASK_CATEGORY)); + ASSERT(lcnt_initialization_completed__); -void erts_lcnt_thread_exit_handler() { - erts_lcnt_thread_data_t *eltd; + changed_categories = (lcnt_category_mask__ ^ mask); + lcnt_category_mask__ = mask; - eltd = ethr_tsd_get(lcnt_thr_data_key); + if(changed_categories) { + lcnt_update_static_locks(); + } - if (eltd) { - free(eltd); + if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION) { + erts_lcnt_update_distribution_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); } -} -/* bindings for bifs */ + if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) { + erts_lcnt_update_allocator_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); + } -Uint16 erts_lcnt_set_rt_opt(Uint16 opt) { - Uint16 prev; - prev = (erts_lcnt_rt_options & opt); - erts_lcnt_rt_options |= opt; - return prev; + if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_PROCESS) { + erts_lcnt_update_process_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_PROCESS); + } + + if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_IO) { + erts_lcnt_update_cio_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); + erts_lcnt_update_driver_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); + erts_lcnt_update_port_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_IO); + } + + if(changed_categories & ERTS_LOCK_FLAGS_CATEGORY_DB) { + erts_lcnt_update_db_locks(mask & ERTS_LOCK_FLAGS_CATEGORY_DB); + } +} + +void erts_lcnt_set_preserve_info(int enable) { + lcnt_preserve_info = enable; } -Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { - Uint16 prev; - prev = (erts_lcnt_rt_options & opt); - erts_lcnt_rt_options &= ~opt; - return prev; +int erts_lcnt_get_preserve_info() { + return lcnt_preserve_info; } void erts_lcnt_clear_counters(void) { - erts_lcnt_lock_t *lock; - erts_lcnt_lock_list_t *list; - erts_lcnt_lock_stats_t *stats; - int i; + erts_lcnt_lock_info_t *iterator; - lcnt_lock(); + lcnt_time__(&lcnt_timer_start); - list = erts_lcnt_data->current_locks; + iterator = NULL; + while(erts_lcnt_iterate_list(&lcnt_current_lock_list, &iterator)) { + lcnt_clear_stats(iterator); + } - for (lock = list->head; lock != NULL; lock = lock->next) { - for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - stats = &lock->stats[i]; - lcnt_clear_stats(stats); - } - lock->n_stats = 1; + iterator = NULL; + while(erts_lcnt_iterate_list(&lcnt_deleted_lock_list, &iterator)) { + erts_lcnt_release_lock_info(iterator); } +} - lock = erts_lcnt_data->deleted_locks->head; - erts_lcnt_data->deleted_locks->head = NULL; - erts_lcnt_data->deleted_locks->tail = NULL; - erts_lcnt_data->deleted_locks->n = 0; +erts_lcnt_data_t erts_lcnt_get_data(void) { + erts_lcnt_time_t timer_stop; + erts_lcnt_data_t result; - lcnt_time(&timer_start); + lcnt_time__(&timer_stop); - lcnt_unlock(); + result.timer_start = lcnt_timer_start; - /* free deleted locks */ - lcnt_list_free(lock); + result.current_locks = &lcnt_current_lock_list; + result.deleted_locks = &lcnt_deleted_lock_list; + + lcnt_time_diff__(&result.duration, &timer_stop, &result.timer_start); + + return result; } -erts_lcnt_data_t *erts_lcnt_get_data(void) { - erts_lcnt_time_t timer_stop; +int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator) { + erts_lcnt_lock_info_t *current, *next; - lcnt_lock(); + current = *iterator ? *iterator : &list->head; - lcnt_time(&timer_stop); - lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start); + ASSERT(current != &list->tail); - lcnt_unlock(); + lcnt_lock_list_entry(current); - return erts_lcnt_data; -} + next = current->next; + + if(next != &list->tail) { + erts_lcnt_retain_lock_info(next); + } + + lcnt_unlock_list_entry(current); + + if(current != &list->head) { + erts_lcnt_release_lock_info(current); + } + + *iterator = next; -char *erts_lcnt_lock_type(Uint16 type) { - return lcnt_lock_type(type); + return next != &list->tail; } -#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ +#endif /* #ifdef ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 6caffbfe86..89d95a73cf 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -18,64 +18,51 @@ * %CopyrightEnd% */ -/* - * Description: Statistics for locks. - * - * Author: Björn-Egil Dahlberg - * Date: 2008-07-03 - * Abstract: - * Locks statistics internal representation. - * - * Conceptual representation, - * - set name - * | - id (the unique lock) - * | | - lock type - * | | - statistics - * | | | - location (file and line number) - * | | | - tries - * | | | - collisions (including trylock busy) - * | | | - timer (time spent in waiting for lock) - * | | | - n_timer (collisions excluding trylock busy) - * | | | - histogram - * | | | | - # 0 = log2(lock wait_time ns) - * | | | | - ... - * | | | | - # n = log2(lock wait_time ns) - * - * Each instance of a lock is the unique lock, i.e. set and id in that set. - * For each lock there is a set of statistics with where and what impact - * the lock aqusition had. - * - * Runtime options - * - suspend, used when internal lock-counting can't be applied. For instance - * when allocating a term for the outside and halloc needs to be used. - * Default: off. - * - location, reserved and not used. - * - proclock, disable proclock counting. Used when performance might be an - * issue. Accessible from erts_debug:lock_counters({process_locks, bool()}). - * Default: off. - * - copysave, enable saving of destroyed locks (and thereby its statistics). - * If memory constraints is an issue this need to be disabled. - * Accessible from erts_debug:lock_counters({copy_save, bool()}). - * Default: off. +/** + * @description Statistics for locks. + * @file erl_lock_count.h + * + * @author Björn-Egil Dahlberg + * @author John Högberg + * + * Conceptual representation: * + * - set name + * | - id (the unique lock) + * | | - lock type + * | | - statistics + * | | | - location (file and line number) + * | | | - attempts + * | | | - collisions (including trylock busy) + * | | | - timer (time spent in waiting for lock) + * | | | - n_timer (collisions excluding trylock busy) + * | | | - histogram + * | | | | - # 0 = log2(lock wait_time ns) + * | | | | - ... + * | | | | - # n = log2(lock wait_time ns) + * + * Each instance of a lock is the unique lock, i.e. set and id in that set. + * For each lock there is a set of statistics with where and what impact + * the lock acquisition had. */ -#include "sys.h" - #ifndef ERTS_LOCK_COUNT_H__ #define ERTS_LOCK_COUNT_H__ #ifdef ERTS_ENABLE_LOCK_COUNT #ifndef ERTS_ENABLE_LOCK_POSITION -/* Enable in order for _x variants of mtx functions to be used. */ +/** @brief Controls whether _x variants of mtx functions are used. */ #define ERTS_ENABLE_LOCK_POSITION 1 #endif +#include "sys.h" #include "ethread.h" -#define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) +#include "erl_term.h" +#include "erl_lock_flags.h" + +#define ERTS_LCNT_MAX_LOCK_LOCATIONS (5) -/* histogram */ #define ERTS_LCNT_HISTOGRAM_MAX_NS (((unsigned long)1LL << 28) - 1) #if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT) #define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30) @@ -85,154 +72,857 @@ #define ERTS_LCNT_HISTOGRAM_RSHIFT (10) #endif -#define ERTS_LCNT_LT_SPINLOCK (((Uint16) 1) << 0) -#define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1) -#define ERTS_LCNT_LT_MUTEX (((Uint16) 1) << 2) -#define ERTS_LCNT_LT_RWMUTEX (((Uint16) 1) << 3) -#define ERTS_LCNT_LT_PROCLOCK (((Uint16) 1) << 4) -#define ERTS_LCNT_LT_ALLOC (((Uint16) 1) << 5) +typedef struct { + unsigned long s; + unsigned long ns; +} erts_lcnt_time_t; + +typedef struct { + /* @brief log2 array of nano seconds occurences */ + Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; +} erts_lcnt_hist_t; -#define ERTS_LCNT_LO_READ (((Uint16) 1) << 6) -#define ERTS_LCNT_LO_WRITE (((Uint16) 1) << 7) +typedef struct { + /** @brief In which file the lock was taken. May be NULL. */ + const char *file; + /** @brief Line number in \c file */ + unsigned int line; -#define ERTS_LCNT_LT_DISABLE (((Uint16) 1) << 8) + /* "attempts" and "collisions" need to be atomic since try_lock busy does + * not acquire a lock and there is no post action to rectify the + * situation. */ -#define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ \ - | ERTS_LCNT_LO_WRITE ) + ethr_atomic_t attempts; + ethr_atomic_t collisions; -#define ERTS_LCNT_LT_ALL ( ERTS_LCNT_LT_SPINLOCK \ - | ERTS_LCNT_LT_RWSPINLOCK \ - | ERTS_LCNT_LT_MUTEX \ - | ERTS_LCNT_LT_RWMUTEX \ - | ERTS_LCNT_LT_PROCLOCK ) + erts_lcnt_time_t total_time_waited; + Uint64 times_waited; -#define ERTS_LCNT_LOCK_TYPE(lock) ((lock)->flag & ERTS_LCNT_LT_ALL) -#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL)) -#define ERTS_LCNT_CLEAR_FLAG(lock) ((lock)->flag = 0) + erts_lcnt_hist_t wait_time_histogram; +} erts_lcnt_lock_stats_t; -/* runtime options */ +typedef struct lcnt_lock_info_t_ { + erts_lock_flags_t flags; + const char *name; + /** @brief Id if possible, must be an immediate */ + Eterm id; -#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) -#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) -#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) -#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3) -#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4) + /* The first entry is reserved as a fallback for when location information + * is missing, and when the lock is used in more than (MAX_LOCK_LOCATIONS + * - 1) different places. */ + erts_lcnt_lock_stats_t location_stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; + unsigned int location_count; -typedef struct { - unsigned long s; - unsigned long ns; -} erts_lcnt_time_t; + /* -- Everything below is internal to this module ---------------------- */ + + /* Lock states; rw locks uses both states, other locks only uses w_state */ -extern erts_lcnt_time_t timer_start; + /** @brief Write state. 0 = not taken, otherwise n threads waiting */ + ethr_atomic_t w_state; + /** @brief Read state. 0 = not taken, > 0 -> writes will wait */ + ethr_atomic_t r_state; + + struct lcnt_lock_info_t_ *prev; + struct lcnt_lock_info_t_ *next; + + /** @brief Used in place of erts_refc_t to avoid a circular dependency. */ + ethr_atomic_t ref_count; + ethr_atomic32_t lock; + + /** @brief Deletion hook called once \c ref_count reaches 0; may defer + * deletion by modifying \c ref_count. */ + void (*dispose)(struct lcnt_lock_info_t_ *); + + struct lcnt_lock_info_carrier_ *carrier; +} erts_lcnt_lock_info_t; + +typedef struct lcnt_lock_info_list_ { + erts_lcnt_lock_info_t head; + erts_lcnt_lock_info_t tail; +} erts_lcnt_lock_info_list_t; typedef struct { - Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ -} erts_lcnt_hist_t; + erts_lcnt_time_t timer_start; /**< Time of last clear */ + erts_lcnt_time_t duration; /**< Time since last clear */ -typedef struct erts_lcnt_lock_stats_s { - /* "tries" and "colls" needs to be atomic since - * trylock busy does not acquire a lock and there - * is no post action to rectify the situation - */ + erts_lcnt_lock_info_list_t *current_locks; + erts_lcnt_lock_info_list_t *deleted_locks; +} erts_lcnt_data_t; - char *file; /* which file the lock was taken */ - unsigned int line; /* line number in file */ +typedef struct lcnt_lock_info_carrier_ erts_lcnt_lock_info_carrier_t; - ethr_atomic_t tries; /* n tries to get lock */ - ethr_atomic_t colls; /* n collisions of tries to get lock */ +typedef ethr_atomic_t erts_lcnt_ref_t; - unsigned long timer_n; /* #times waited for lock */ - erts_lcnt_time_t timer; /* total wait time for lock */ - erts_lcnt_hist_t hist; -} erts_lcnt_lock_stats_t; +/* -- Globals -------------------------------------------------------------- */ + +/** @brief Checks whether counting is enabled for any of the given + * categories. */ +#define erts_lcnt_check_enabled(flags) \ + (lcnt_category_mask__ & flags) + +/* -- Lock operations ------------------------------------------------------ + * + * All of these will nop if there's nothing "installed" on the given reference, + * in order to transparently support enable/disable at runtime. */ + +/** @brief Records that a lock is being acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_lock + * @param option Notes whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option); + +/** @brief Records that a lock has been acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_lock_post. + * @param file The name of the file where the lock was acquired. + * @param line The line at which the lock was acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line); + +/** @brief Records that a lock has been released. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_unlock_opt + * @param option Whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option); + +/** @brief Rectifies the case where a lock wasn't actually a lock operation. + * + * Only used for process locks at the moment. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref); + +/** @brief Records the result of a trylock, placing the queried lock status in + * \c result. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result); + +/** @copydoc erts_lcnt_trylock + * @param option Whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, erts_lock_options_t option); + +/* Indexed variants of the standard lock operations, for use when a single + * reference contains many counters (eg. process locks). + * + * erts_lcnt_open_ref must be used to safely extract the installed carrier, + * which must released with erts_lcnt_close_reference on success. + * + * Refer to \c erts_lcnt_lock for example usage. */ + +ERTS_GLB_INLINE +void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option); + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line); + +ERTS_GLB_INLINE +void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); + +ERTS_GLB_INLINE +void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option); + +ERTS_GLB_INLINE +void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result); +ERTS_GLB_INLINE +void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, erts_lock_options_t option); + +/* -- Reference operations ------------------------------------------------- */ + +/** @brief Registers a lock counter reference; this must be called prior to + * using any other functions in this module. */ +ERTS_GLB_INLINE +void erts_lcnt_init_ref(erts_lcnt_ref_t *ref); + +/** @brief As \c erts_lcnt_init_ref, but also enables lock counting right + * away if appropriate to reduce noise. + * @param id An immediate erlang term with whatever extra data you want to + * identify this lock with. */ +ERTS_GLB_INLINE +void erts_lcnt_init_ref_x(erts_lcnt_ref_t *ref, const char *name, + Eterm id, erts_lock_flags_t flags); + +/** @brief Checks whether counting is enabled on the given reference. */ +ERTS_GLB_FORCE_INLINE +int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref); + +/** @brief Convenience macro to re/enable counting on an already initialized + * reference. Don't forget to specify the lock type in \c flags! */ +#define erts_lcnt_install_new_lock_info(ref, name, id, flags) \ + if(!erts_lcnt_check_ref_installed(ref)) { \ + erts_lcnt_lock_info_carrier_t *__carrier; \ + __carrier = erts_lcnt_create_lock_info_carrier(1);\ + erts_lcnt_init_lock_info_idx(__carrier, 0, name, id, flags); \ + erts_lcnt_install(ref, __carrier);\ + } while(0) + +erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count); + +/* @brief Initializes the lock info at the given index. + * @param id An immediate erlang term with whatever extra data you want to + * identify this lock with. + * @param flags The flags the lock itself was initialized with. Keep in mind + * that all locks in a carrier must share the same category/static property. */ +ERTS_GLB_INLINE +void erts_lcnt_init_lock_info_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, + const char *name, Eterm id, erts_lock_flags_t flags); + +/** @brief Atomically installs the given lock counters. Nops (and releases the + * provided carrier) if something was already installed. */ +void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier); + +/** @brief Atomically removes the currently installed lock counters. Nops if + * nothing was installed. */ +void erts_lcnt_uninstall(erts_lcnt_ref_t *ref); + +ERTS_GLB_FORCE_INLINE +int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result); + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier); + +/* -- Module initialization ------------------------------------------------ */ + +void erts_lcnt_pre_thr_init(void); +void erts_lcnt_post_thr_init(void); +void erts_lcnt_late_init(void); + +/* @brief Called after everything in the system has been initialized, including + * the schedulers. This is mainly a backwards compatibility shim for matching + * the old lcnt behavior where all lock counting was enabled by default. */ +void erts_lcnt_post_startup(void); + +void erts_lcnt_thread_setup(void); +void erts_lcnt_thread_exit_handler(void); + +/* -- BIF interface -------------------------------------------------------- */ -/* rw locks uses both states, other locks only uses w_state */ -typedef struct erts_lcnt_lock_s { - char *name; /* lock name */ - Uint16 flag; /* lock type */ - Eterm id; /* id if possible */ +/** @brief Safely iterates through all entries in the given list. + * + * The referenced item will be valid until the next call to + * \c erts_lcnt_iterate_list after which point it may be destroyed; call + * erts_lcnt_retain_lock_info if you wish to hang on to it beyond that point. + * + * Iteration can be cancelled by calling erts_lcnt_release_lock_info on the + * iterator and breaking out of the loop. + * + * @param iterator The iteration variable; set the pointee to NULL to start + * iteration. + * @return 1 while the iterator is valid, 0 at the end of the list. */ +int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator); + +/** @brief Clears the counter state of all locks, and releases all locks + * preserved through erts_lcnt_set_preserve_info (if any). */ +void erts_lcnt_clear_counters(void); + +/** @brief Retrieves the global lock counter state. + * + * Note that the lists may be modified while you're mucking around with them. + * Always use \c erts_lcnt_iterate_list to enumerate them. */ +erts_lcnt_data_t erts_lcnt_get_data(void); + +void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info); +void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info); + +/** @brief Sets whether to preserve the info of destroyed/uninstalled locks. + * + * This option makes no distinction whether the lock was destroyed or if lock + * counting was simply disabled, so erts_lcnt_set_category_mask must not be + * used while this option is active. */ +void erts_lcnt_set_preserve_info(int enable); + +int erts_lcnt_get_preserve_info(void); + +/** @brief Updates the category mask, enabling or disabling counting on the + * affected locks as necessary. + * + * This is not guaranteed to find all existing locks; only those that are + * flagged as static locks and those reachable through other means can be + * altered. */ +void erts_lcnt_set_category_mask(erts_lock_flags_t mask); + +erts_lock_flags_t erts_lcnt_get_category_mask(void); + +/* -- Inline implementation ------------------------------------------------ */ + +/* The following is a hack to get the things we need from erl_thr_progress.h, + * which we can't #include without dependency hell breaking loose. + * + * The size of LcntThrPrgrLaterOp and value of the constant are verified at + * compile-time in erts_lcnt_pre_thr_init. */ + +int lcnt_thr_progress_unmanaged_delay__(void); +void lcnt_thr_progress_unmanaged_continue__(int handle); +typedef struct { Uint64 _[4]; } LcntThrPrgrLaterOp; +#define LCNT_THR_PRGR_DHANDLE_MANAGED -1 + +struct lcnt_lock_info_carrier_ { + ethr_atomic_t ref_count; + + LcntThrPrgrLaterOp release_entries; + + unsigned char entry_count; + erts_lcnt_lock_info_t entries[]; +}; + +typedef struct { + erts_lcnt_time_t timer; /* timer */ + int timer_set; /* bool */ + int lock_in_conflict; /* bool */ +} lcnt_thread_data_t__; + +extern const int lcnt_log2_tab64__[]; + +extern ethr_tsd_key lcnt_thr_data_key__; +extern erts_lock_flags_t lcnt_category_mask__; #ifdef DEBUG - ethr_atomic_t flowstate; +extern int lcnt_initialization_completed__; #endif - /* lock states */ - ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */ - ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */ +void lcnt_register_static_lock__(erts_lcnt_ref_t *reference, const char *name, Eterm id, + erts_lock_flags_t flags); - /* statistics */ - unsigned int n_stats; - erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/ +void lcnt_deallocate_carrier__(erts_lcnt_lock_info_carrier_t *carrier); - /* chains for list handling */ - /* data is hold by lcnt_lock */ - struct erts_lcnt_lock_s *prev; - struct erts_lcnt_lock_s *next; -} erts_lcnt_lock_t; +ERTS_GLB_INLINE +int lcnt_log2__(Uint64 v); -typedef struct { - erts_lcnt_lock_t *head; - erts_lcnt_lock_t *tail; - unsigned long n; -} erts_lcnt_lock_list_t; +ERTS_GLB_INLINE +void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited); -typedef struct { - erts_lcnt_time_t duration; /* time since last clear */ - erts_lcnt_lock_list_t *current_locks; - erts_lcnt_lock_list_t *deleted_locks; -} erts_lcnt_data_t; +ERTS_GLB_INLINE +void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited); -typedef struct { - int id; +ERTS_GLB_INLINE +erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line); - erts_lcnt_time_t timer; /* timer */ - int timer_set; /* bool */ - int lock_in_conflict; /* bool */ -} erts_lcnt_thread_data_t; +ERTS_GLB_INLINE +void lcnt_dec_lock_state__(ethr_atomic_t *l_state); -/* globals */ +ERTS_GLB_INLINE +void lcnt_time__(erts_lcnt_time_t *time); -extern Uint16 erts_lcnt_rt_options; +ERTS_GLB_INLINE +void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d); -/* function declerations */ +ERTS_GLB_INLINE +void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0); -void erts_lcnt_init(void); -void erts_lcnt_late_init(void); +ERTS_GLB_INLINE +void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier); -/* thread operations */ -void erts_lcnt_thread_setup(void); -void erts_lcnt_thread_exit_handler(void); +ERTS_GLB_INLINE +void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier); + +ERTS_GLB_INLINE +lcnt_thread_data_t__ *lcnt_get_thread_data__(void); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +void lcnt_time__(erts_lcnt_time_t *time) { + /* + * erts_sys_hrtime() is the highest resolution + * we could find, it may or may not be monotonic... + */ + ErtsMonotonicTime mtime = erts_sys_hrtime(); + time->s = (unsigned long) (mtime / 1000000000LL); + time->ns = (unsigned long) (mtime - 1000000000LL*time->s); +} + +/* difference d must be non-negative */ + +ERTS_GLB_INLINE +void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { + t->s += d->s; + t->ns += d->ns; + + t->s += t->ns / 1000000000LL; + t->ns = t->ns % 1000000000LL; +} + +ERTS_GLB_INLINE +void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { + long ds; + long dns; + + ds = t1->s - t0->s; + dns = t1->ns - t0->ns; + + /* the difference should not be able to get bigger than 1 sec in ns*/ + + if (dns < 0) { + ds -= 1; + dns += 1000000000LL; + } + + ASSERT(ds >= 0); + + d->s = ds; + d->ns = dns; +} + +ERTS_GLB_INLINE +int lcnt_log2__(Uint64 v) { + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + + return lcnt_log2_tab64__[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; +} + +ERTS_GLB_INLINE +void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited) { + int idx; + + if(time_waited->s > 0 || time_waited->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { + idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; + } else { + unsigned long r = time_waited->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; + + idx = r ? lcnt_log2__(r) : 0; + } + + hist->ns[idx]++; +} + +ERTS_GLB_INLINE +void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited) { + ethr_atomic_inc(&stats->attempts); + + if(lock_in_conflict) { + ethr_atomic_inc(&stats->collisions); + } + + if(time_waited) { + stats->times_waited++; + + lcnt_time_add__(&stats->total_time_waited, time_waited); + lcnt_update_wait_histogram__(&stats->wait_time_histogram, time_waited); + } +} + +/* If we were installed while the lock was held, r/w_state will be 0 and we + * can't tell which unlock or unacquire operation was the last. To get around + * this we assume that all excess operations go *towards* zero rather than down + * to zero, eventually becoming consistent with the actual state once the lock + * is fully released. + * + * Conflicts might not be counted until the recorded state is fully consistent + * with the actual state, but there should be no other ill effects. */ + +ERTS_GLB_INLINE +void lcnt_dec_lock_state__(ethr_atomic_t *l_state) { + ethr_sint_t state = ethr_atomic_dec_read_acqb(l_state); + + /* We can not assume that state is >= -1 here; unlock and unacquire might + * bring it below -1 and race to increment it back. */ + + if(state < 0) { + ethr_atomic_inc_acqb(l_state); + } +} + +ERTS_GLB_INLINE +erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line) { + unsigned int i; + + ASSERT(info->location_count >= 1 && info->location_count <= ERTS_LCNT_MAX_LOCK_LOCATIONS); + + for(i = 0; i < info->location_count; i++) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[i]; + + if(stats->file == file && stats->line == line) { + return stats; + } + } -/* list operations (local) */ -erts_lcnt_lock_list_t *erts_lcnt_list_init(void); + if(info->location_count < ERTS_LCNT_MAX_LOCK_LOCATIONS) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[info->location_count]; -void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); -void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); + stats->file = file; + stats->line = line; -/* lock operations (global) */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag); -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id); -void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock); -void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock); + info->location_count++; -void erts_lcnt_lock(erts_lcnt_lock_t *lock); -void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option); -void erts_lcnt_lock_post(erts_lcnt_lock_t *lock); -void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line); -void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock); + return stats; + } -void erts_lcnt_unlock(erts_lcnt_lock_t *lock); -void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option); + return &info->location_stats[0]; +} -void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option); -void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res); +ERTS_GLB_INLINE +lcnt_thread_data_t__ *lcnt_get_thread_data__(void) { + lcnt_thread_data_t__ *eltd = (lcnt_thread_data_t__ *)ethr_tsd_get(lcnt_thr_data_key__); -/* bif interface */ -Uint16 erts_lcnt_set_rt_opt(Uint16 opt); -Uint16 erts_lcnt_clear_rt_opt(Uint16 opt); -void erts_lcnt_clear_counters(void); -char *erts_lcnt_lock_type(Uint16 type); -erts_lcnt_data_t *erts_lcnt_get_data(void); + ASSERT(eltd); + + return eltd; +} + +ERTS_GLB_FORCE_INLINE +int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result) { + if(ERTS_LIKELY(!erts_lcnt_check_ref_installed(ref))) { + return 0; + } + + ASSERT(lcnt_initialization_completed__); + + (*handle) = lcnt_thr_progress_unmanaged_delay__(); + (*result) = (erts_lcnt_lock_info_carrier_t*)ethr_atomic_read(ref); + + if(*result) { + if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_retain_carrier__(*result); + lcnt_thr_progress_unmanaged_continue__(*handle); + } + + return 1; + } else if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_thr_progress_unmanaged_continue__(*handle); + } + + return 0; +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier) { + if(handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_release_carrier__(carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_init_ref(erts_lcnt_ref_t *ref) { + ethr_atomic_init(ref, (ethr_sint_t)NULL); +} + +ERTS_GLB_INLINE +void erts_lcnt_init_ref_x(erts_lcnt_ref_t *ref, const char *name, + Eterm id, erts_lock_flags_t flags) { + erts_lcnt_init_ref(ref); + + if(flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC) { + lcnt_register_static_lock__(ref, name, id, flags); + } + + if(erts_lcnt_check_enabled(flags)) { + erts_lcnt_install_new_lock_info(ref, name, id, flags); + } +} + +ERTS_GLB_FORCE_INLINE +int erts_lcnt_check_ref_installed(erts_lcnt_ref_t *ref) { + return (!!*ethr_atomic_addr(ref)); +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_opt_idx(carrier, 0, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_post_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_post_x_idx(carrier, 0, file, line); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_unacquire_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_unlock_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, erts_lock_options_t option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_unlock_opt_idx(carrier, 0, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_trylock_idx(carrier, 0, result); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, erts_lock_options_t option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_trylock_opt_idx(carrier, 0, result, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_opt_idx(carrier, index, ERTS_LOCK_OPTIONS_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE)); + + if(option & ERTS_LOCK_OPTIONS_WRITE) { + ethr_sint_t w_state, r_state; + + w_state = ethr_atomic_inc_read(&info->w_state) - 1; + r_state = ethr_atomic_read(&info->r_state); + + /* We cannot acquire w_lock if either w or r are taken */ + eltd->lock_in_conflict = (w_state > 0) || (r_state > 0); + } else { + ethr_sint_t w_state = ethr_atomic_read(&info->w_state); + + /* We cannot acquire r_lock if w_lock is taken */ + eltd->lock_in_conflict = (w_state > 0); + } + + if(option & ERTS_LOCK_OPTIONS_READ) { + ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE); + ethr_atomic_inc(&info->r_state); + } + + if(eltd->lock_in_conflict) { + /* Only set the timer if nobody else has it. This should only happen + * when proc_locks acquires several locks "atomically." All other locks + * will block the thread when locked (w_state > 0) */ + if(eltd->timer_set == 0) { + lcnt_time__(&eltd->timer); + } + + eltd->timer_set++; + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_post_x_idx(carrier, index, NULL, 0); +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); + erts_lcnt_lock_stats_t *stats; + + ASSERT(index < carrier->entry_count); + + /* If the lock was in conflict, update the time spent waiting. */ + stats = lcnt_get_lock_stats__(info, file, line); + if(eltd->timer_set) { + erts_lcnt_time_t time_wait; + erts_lcnt_time_t timer; + + lcnt_time__(&timer); + + lcnt_time_diff__(&time_wait, &timer, &eltd->timer); + lcnt_update_stats__(stats, eltd->lock_in_conflict, &time_wait); + + eltd->timer_set--; + + ASSERT(eltd->timer_set >= 0); + } else { + lcnt_update_stats__(stats, eltd->lock_in_conflict, NULL); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + ASSERT(index < carrier->entry_count); + + erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LOCK_OPTIONS_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, erts_lock_options_t option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE)); + + if(option & ERTS_LOCK_OPTIONS_WRITE) { + lcnt_dec_lock_state__(&info->w_state); + } + + if(option & ERTS_LOCK_OPTIONS_READ) { + ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE); + lcnt_dec_lock_state__(&info->r_state); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + lcnt_dec_lock_state__(&info->w_state); +} + +ERTS_GLB_INLINE +void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result) { + ASSERT(index < carrier->entry_count); + + erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LOCK_OPTIONS_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, erts_lock_options_t option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LOCK_OPTIONS_READ) || (option & ERTS_LOCK_OPTIONS_WRITE)); + + if(result != EBUSY) { + if(option & ERTS_LOCK_OPTIONS_WRITE) { + ethr_atomic_inc(&info->w_state); + } + + if(option & ERTS_LOCK_OPTIONS_READ) { + ASSERT(info->flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE); + ethr_atomic_inc(&info->r_state); + } + + lcnt_update_stats__(&info->location_stats[0], 0, NULL); + } else { + ethr_atomic_inc(&info->location_stats[0].attempts); + ethr_atomic_inc(&info->location_stats[0].collisions); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_init_lock_info_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, + const char *name, Eterm id, erts_lock_flags_t flags) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(is_immed(id)); + + ASSERT(flags & ERTS_LOCK_FLAGS_MASK_TYPE); + ASSERT(flags & ERTS_LOCK_FLAGS_MASK_CATEGORY); + + info->flags = flags; + info->name = name; + info->id = id; +} + +ERTS_GLB_INLINE +void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier) { +#ifdef DEBUG + ASSERT(ethr_atomic_inc_read_acqb(&carrier->ref_count) >= 2); +#else + ethr_atomic_inc_acqb(&carrier->ref_count); +#endif +} + +ERTS_GLB_INLINE +void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier) { + ethr_sint_t count = ethr_atomic_dec_read_relb(&carrier->ref_count); + + ASSERT(count >= 0); + + if(count == 0) { + lcnt_deallocate_carrier__(carrier); + } +} + +#endif #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ #endif /* ifndef ERTS_LOCK_COUNT_H__ */ diff --git a/erts/emulator/beam/erl_lock_flags.c b/erts/emulator/beam/erl_lock_flags.c new file mode 100644 index 0000000000..e0a0e95c09 --- /dev/null +++ b/erts/emulator/beam/erl_lock_flags.c @@ -0,0 +1,59 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "erl_lock_flags.h" + +const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags) { + switch(flags & ERTS_LOCK_FLAGS_MASK_TYPE) { + case ERTS_LOCK_FLAGS_TYPE_PROCLOCK: + return "proclock"; + case ERTS_LOCK_FLAGS_TYPE_MUTEX: + if(flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) { + return "rw_mutex"; + } + + return "mutex"; + case ERTS_LOCK_FLAGS_TYPE_SPINLOCK: + if(flags & ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) { + return "rw_spinlock"; + } + + return "spinlock"; + default: + return "garbage"; + } +} + +const char *erts_lock_options_get_short_desc(erts_lock_options_t options) { + switch(options) { + case ERTS_LOCK_OPTIONS_RDWR: + return "rw"; + case ERTS_LOCK_OPTIONS_READ: + return "r"; + case ERTS_LOCK_OPTIONS_WRITE: + return "w"; + default: + return "none"; + } +} diff --git a/erts/emulator/beam/erl_lock_flags.h b/erts/emulator/beam/erl_lock_flags.h new file mode 100644 index 0000000000..d711f69456 --- /dev/null +++ b/erts/emulator/beam/erl_lock_flags.h @@ -0,0 +1,78 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERTS_LOCK_FLAGS_H__ +#define ERTS_LOCK_FLAGS_H__ + +#define ERTS_LOCK_OPTIONS_READ (1 << 1) +#define ERTS_LOCK_OPTIONS_WRITE (1 << 2) + +#define ERTS_LOCK_OPTIONS_RDWR (ERTS_LOCK_OPTIONS_READ | ERTS_LOCK_OPTIONS_WRITE) + +/* Property/category are bitfields to simplify their use in masks. */ +#define ERTS_LOCK_FLAGS_MASK_CATEGORY (0xFFC0) +#define ERTS_LOCK_FLAGS_MASK_PROPERTY (0x0030) + +/* Type is a plain number. */ +#define ERTS_LOCK_FLAGS_MASK_TYPE (0x000F) + +#define ERTS_LOCK_FLAGS_TYPE_SPINLOCK (1) +#define ERTS_LOCK_FLAGS_TYPE_MUTEX (2) +#define ERTS_LOCK_FLAGS_TYPE_PROCLOCK (3) + +/* "Static" guarantees that the lock will never be destroyed once created. */ +#define ERTS_LOCK_FLAGS_PROPERTY_STATIC (1 << 4) +#define ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE (1 << 5) + +#define ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR (1 << 6) +#define ERTS_LOCK_FLAGS_CATEGORY_PROCESS (1 << 7) +#define ERTS_LOCK_FLAGS_CATEGORY_IO (1 << 8) +#define ERTS_LOCK_FLAGS_CATEGORY_DB (1 << 9) +#define ERTS_LOCK_FLAGS_CATEGORY_DEBUG (1 << 10) +#define ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER (1 << 11) +#define ERTS_LOCK_FLAGS_CATEGORY_GENERIC (1 << 12) +#define ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION (1 << 13) + +#define ERTS_LOCK_TYPE_SPINLOCK \ + (ERTS_LOCK_FLAGS_TYPE_SPINLOCK) +#define ERTS_LOCK_TYPE_RWSPINLOCK \ + (ERTS_LOCK_TYPE_SPINLOCK | \ + ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) +#define ERTS_LOCK_TYPE_MUTEX \ + (ERTS_LOCK_FLAGS_TYPE_MUTEX) +#define ERTS_LOCK_TYPE_RWMUTEX \ + (ERTS_LOCK_TYPE_MUTEX | \ + ERTS_LOCK_FLAGS_PROPERTY_READ_WRITE) +#define ERTS_LOCK_TYPE_PROCLOCK \ + (ERTS_LOCK_FLAGS_CATEGORY_PROCESS | \ + ERTS_LOCK_FLAGS_TYPE_PROCLOCK) + +/* -- -- */ + +typedef unsigned short erts_lock_flags_t; +typedef unsigned short erts_lock_options_t; + +/* @brief Gets the type name of the lock, honoring the RW flag if supplied. */ +const char *erts_lock_flags_get_type_name(erts_lock_flags_t flags); + +/* @brief Gets a short-form description of the given lock options. (rw/r/w) */ +const char *erts_lock_options_get_short_desc(erts_lock_options_t options); + +#endif /* ERTS_LOCK_FLAGS_H__ */ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index c1af70592a..abf194cf94 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -170,7 +170,7 @@ erts_cleanup_offheap(ErlOffHeap *offheap) erts_bin_release(u.pb->val); break; case FUN_SUBTAG: - if (erts_smp_refc_dectest(&u.fun->fe->refc, 0) == 0) { + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { erts_erase_fun_entry(u.fun->fe); } break; @@ -265,11 +265,9 @@ erts_queue_dist_message(Process *rcvr, Sint tok_lastcnt = 0; Sint tok_serial = 0; #endif -#ifdef ERTS_SMP erts_aint_t state; -#endif - ERTS_SMP_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); + ERTS_LC_ASSERT(rcvr_locks == erts_proc_lc_my_proc_locks(rcvr)); mp = erts_alloc_message(0, NULL); mp->data.dist_ext = dist_ext; @@ -283,36 +281,34 @@ erts_queue_dist_message(Process *rcvr, #endif ERL_MESSAGE_TOKEN(mp) = token; -#ifdef ERTS_SMP if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) { - if (erts_smp_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + if (erts_proc_trylock(rcvr, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks = ERTS_PROC_LOCK_MSGQ; ErtsProcLocks unlocks = rcvr_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ); if (unlocks) { - erts_smp_proc_unlock(rcvr, unlocks); + erts_proc_unlock(rcvr, unlocks); need_locks |= unlocks; } - erts_smp_proc_lock(rcvr, need_locks); + erts_proc_lock(rcvr, need_locks); } } - state = erts_smp_atomic32_read_acqb(&rcvr->state); + state = erts_atomic32_read_acqb(&rcvr->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) - erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); /* Drop message if receiver is exiting or has a pending exit ... */ erts_cleanup_messages(mp); } else -#endif if (IS_TRACED_FL(rcvr, F_TRACE_RECEIVE)) { if (from == am_Empty) from = dist_ext->dep->sysname; /* Ahh... need to decode it in order to trace it... */ if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) - erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); if (!erts_decode_dist_message(rcvr, rcvr_locks, mp, 0)) erts_free_message(mp); else { @@ -361,14 +357,10 @@ erts_queue_dist_message(Process *rcvr, LINK_MESSAGE(rcvr, mp, &mp->next, 1); if (!(rcvr_locks & ERTS_PROC_LOCK_MSGQ)) - erts_smp_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(rcvr, ERTS_PROC_LOCK_MSGQ); erts_proc_notify_new_message(rcvr, -#ifdef ERTS_SMP rcvr_locks -#else - 0 -#endif ); } } @@ -393,50 +385,45 @@ queue_messages(Process* receiver, ERL_MESSAGE_TOKEN(first) == NIL || is_tuple(ERL_MESSAGE_TOKEN(first))); -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ || + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(receiver) < ERTS_PROC_LOCK_MSGQ || receiver_locks == erts_proc_lc_my_proc_locks(receiver)); #endif if (!(receiver_locks & ERTS_PROC_LOCK_MSGQ)) { - if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + if (erts_proc_trylock(receiver, ERTS_PROC_LOCK_MSGQ) == EBUSY) { ErtsProcLocks need_locks; if (receiver_state) state = *receiver_state; else - state = erts_smp_atomic32_read_nob(&receiver->state); + state = erts_atomic32_read_nob(&receiver->state); if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) goto exiting; need_locks = receiver_locks & ERTS_PROC_LOCKS_HIGHER_THAN(ERTS_PROC_LOCK_MSGQ); if (need_locks) { - erts_smp_proc_unlock(receiver, need_locks); + erts_proc_unlock(receiver, need_locks); } need_locks |= ERTS_PROC_LOCK_MSGQ; - erts_smp_proc_lock(receiver, need_locks); + erts_proc_lock(receiver, need_locks); } locked_msgq = 1; } -#endif - state = erts_smp_atomic32_read_nob(&receiver->state); + state = erts_atomic32_read_nob(&receiver->state); if (state & (ERTS_PSFLG_PENDING_EXIT|ERTS_PSFLG_EXITING)) { -#ifdef ERTS_SMP exiting: -#endif /* Drop message if receiver is exiting or has a pending exit... */ if (locked_msgq) - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); erts_cleanup_messages(first); return 0; } res = receiver->msg.len; -#ifdef ERTS_SMP if (receiver_locks & ERTS_PROC_LOCK_MAIN) { /* * We move 'in queue' to 'private queue' and place @@ -447,11 +434,10 @@ queue_messages(Process* receiver, * the root set when garbage collecting. */ res += receiver->msg_inq.len; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(receiver); + ERTS_MSGQ_MV_INQ2PRIVQ(receiver); LINK_MESSAGE_PRIVQ(receiver, first, last, len); } else -#endif { LINK_MESSAGE(receiver, first, last, len); } @@ -489,14 +475,10 @@ queue_messages(Process* receiver, } if (locked_msgq) { - erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); + erts_proc_unlock(receiver, ERTS_PROC_LOCK_MSGQ); } -#ifdef ERTS_SMP erts_proc_notify_new_message(receiver, receiver_locks); -#else - erts_proc_notify_new_message(receiver, 0); -#endif return res; } @@ -568,14 +550,11 @@ erts_msg_attached_data_size_aux(ErtsMessage *msg) sz = erts_decode_dist_ext_size(msg->data.dist_ext); if (sz < 0) { - /* Bad external; remove it */ - if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { - ErlHeapFragment *heap_frag; - heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(msg->data.dist_ext); - msg->data.dist_ext = NULL; + /* Bad external + * We leave the message intact in this case as it's not worth the trouble + * to make all callers remove it from queue. It will be detected again + * and removed from message queue later anyway. + */ return 0; } @@ -597,9 +576,7 @@ erts_try_alloc_message_on_heap(Process *pp, ErlOffHeap **ohpp, int *on_heap_p) { -#ifdef ERTS_SMP int locked_main = 0; -#endif ErtsMessage *mp; ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); @@ -607,15 +584,9 @@ erts_try_alloc_message_on_heap(Process *pp, if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) goto in_message_fragment; else if ( -#if defined(ERTS_SMP) *plp & ERTS_PROC_LOCK_MAIN -#else - pp -#endif ) { -#ifdef ERTS_SMP try_on_heap: -#endif if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) || (pp->flags & F_DISABLE_GC) || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { @@ -623,12 +594,10 @@ erts_try_alloc_message_on_heap(Process *pp, * The heap is either potentially in an inconsistent * state, or not large enough. */ -#ifdef ERTS_SMP if (locked_main) { *plp &= ~ERTS_PROC_LOCK_MAIN; - erts_smp_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(pp, ERTS_PROC_LOCK_MAIN); } -#endif goto in_message_fragment; } @@ -639,14 +608,12 @@ erts_try_alloc_message_on_heap(Process *pp, mp->data.attached = NULL; *on_heap_p = !0; } -#ifdef ERTS_SMP - else if (pp && erts_smp_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { + else if (pp && erts_proc_trylock(pp, ERTS_PROC_LOCK_MAIN) == 0) { locked_main = 1; - *psp = erts_smp_atomic32_read_nob(&pp->state); + *psp = erts_atomic32_read_nob(&pp->state); *plp |= ERTS_PROC_LOCK_MAIN; goto try_on_heap; } -#endif else { in_message_fragment: if (!((*psp) & ERTS_PSFLG_ON_HEAP_MSGQ)) { @@ -715,7 +682,7 @@ erts_send_message(Process* sender, } #endif - receiver_state = erts_smp_atomic32_read_nob(&receiver->state); + receiver_state = erts_atomic32_read_nob(&receiver->state); if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { Eterm* hp; @@ -964,7 +931,7 @@ erts_move_messages_off_heap(Process *c_p) reds += c_p->msg.len / 10; - ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); @@ -1029,9 +996,9 @@ erts_complete_off_heap_message_queue_change(Process *c_p) { int reds = 1; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); ASSERT(c_p->flags & F_OFF_HEAP_MSGQ_CHNG); - ASSERT(erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); + ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); /* * This job was first initiated when the process changed to off heap @@ -1043,13 +1010,13 @@ erts_complete_off_heap_message_queue_change(Process *c_p) */ if (!(c_p->flags & F_OFF_HEAP_MSGQ)) - erts_smp_atomic32_read_band_nob(&c_p->state, + erts_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_OFF_HEAP_MSGQ); else { reds += 2; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); reds += erts_move_messages_off_heap(c_p); } c_p->flags &= ~F_OFF_HEAP_MSGQ_CHNG; @@ -1086,16 +1053,16 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) #ifdef DEBUG if (c_p->flags & F_OFF_HEAP_MSGQ) { - ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); } else { if (c_p->flags & F_OFF_HEAP_MSGQ_CHNG) { - ASSERT(erts_smp_atomic32_read_nob(&c_p->state) + ASSERT(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ); } else { - ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + ASSERT(!(erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ)); } } @@ -1112,7 +1079,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) case am_on_heap: c_p->flags |= F_ON_HEAP_MSGQ; c_p->flags &= ~F_OFF_HEAP_MSGQ; - erts_smp_atomic32_read_bor_nob(&c_p->state, + erts_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_ON_HEAP_MSGQ); /* * We are not allowed to clear ERTS_PSFLG_OFF_HEAP_MSGQ @@ -1121,7 +1088,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) */ if (!(c_p->flags & F_OFF_HEAP_MSGQ_CHNG)) { /* Safe to clear ERTS_PSFLG_OFF_HEAP_MSGQ... */ - erts_smp_atomic32_read_band_nob(&c_p->state, + erts_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_OFF_HEAP_MSGQ); } break; @@ -1139,7 +1106,7 @@ erts_change_message_queue_management(Process *c_p, Eterm new_state) break; case am_off_heap: c_p->flags &= ~F_ON_HEAP_MSGQ; - erts_smp_atomic32_read_band_nob(&c_p->state, + erts_atomic32_read_band_nob(&c_p->state, ~ERTS_PSFLG_ON_HEAP_MSGQ); goto change_to_off_heap; default: @@ -1174,7 +1141,7 @@ change_to_off_heap: * change has completed, GC does not need to inspect * the message queue at all. */ - erts_smp_atomic32_read_bor_nob(&c_p->state, + erts_atomic32_read_bor_nob(&c_p->state, ERTS_PSFLG_OFF_HEAP_MSGQ); c_p->flags |= F_OFF_HEAP_MSGQ_CHNG; cohmq = erts_alloc(ERTS_ALC_T_MSGQ_CHNG, @@ -1455,7 +1422,7 @@ erts_factory_message_create(ErtsHeapFactory* factory, int on_heap; erts_aint32_t state; - state = proc ? erts_smp_atomic32_read_nob(&proc->state) : 0; + state = proc ? erts_atomic32_read_nob(&proc->state) : 0; if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { msgp = erts_alloc_message(sz, &hp); @@ -1470,7 +1437,7 @@ erts_factory_message_create(ErtsHeapFactory* factory, } if (on_heap) { - ERTS_SMP_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN); + ERTS_ASSERT(*proc_locksp & ERTS_PROC_LOCK_MAIN); ASSERT(ohp == &proc->off_heap); factory->mode = FACTORY_HALLOC; factory->p = proc; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 42ed14e69c..9c8cf84e43 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -174,7 +174,6 @@ typedef struct { ErtsMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; -#ifdef ERTS_SMP typedef struct { ErtsMessage* first; @@ -190,7 +189,6 @@ typedef struct erl_trace_message_queue__ { Sint len; /* queue length */ } ErlTraceMessageQueue; -#endif /* Get "current" message */ #define PEEK_MESSAGE(p) (*(p)->msg.save) @@ -207,7 +205,6 @@ typedef struct erl_trace_message_queue__ { (p)->where.len += (num_msgs); \ } while(0) -#ifdef ERTS_SMP /* Add message last in private message queue */ #define LINK_MESSAGE_PRIVQ(p, first_msg, last_msg, len) \ @@ -219,7 +216,7 @@ typedef struct erl_trace_message_queue__ { #define LINK_MESSAGE(p, first_msg, last_msg, len) \ LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg_inq) -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) \ +#define ERTS_MSGQ_MV_INQ2PRIVQ(p) \ do { \ if (p->msg_inq.first) { \ *p->msg.last = p->msg_inq.first; \ @@ -231,17 +228,6 @@ typedef struct erl_trace_message_queue__ { } \ } while (0) -#else - -#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p) - -/* Add message last_msg in message queue */ -#define LINK_MESSAGE(p, first_msg, last_msg, len) \ - do { \ - LINK_MESSAGE_IMPL(p, first_msg, last_msg, len, msg); \ - } while(0) - -#endif /* Unlink current message */ #define UNLINK_MESSAGE(p,msgp) do { \ diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 3994800ba7..1c840d89f6 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -54,7 +54,7 @@ #define DIR_RIGHT 1 #define DIR_END 2 -static erts_smp_atomic_t tot_link_lh_size; +static erts_atomic_t tot_link_lh_size; /* Implements the sort order in monitor trees, which is different from the ordinary term order. @@ -123,7 +123,7 @@ do { \ (*((Hp)++)) = boxed_val((From))[i__]; \ if (is_external((To))) { \ external_thing_ptr((To))->next = NULL; \ - erts_smp_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\ + erts_refc_inc(&(external_thing_ptr((To))->node->refc), 2);\ } \ } \ } while (0) @@ -145,7 +145,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, UWord entity, Eterm nam } else { n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH, mon_size*sizeof(Uint)); - erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint)); + erts_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint)); } hp = n->heap; @@ -179,7 +179,7 @@ static ErtsLink *create_link(Uint type, Eterm pid) } else { n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH, lnk_size*sizeof(Uint)); - erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint)); + erts_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint)); } hp = n->heap; @@ -214,13 +214,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid) void erts_init_monitors(void) { - erts_smp_atomic_init_nob(&tot_link_lh_size, 0); + erts_atomic_init_nob(&tot_link_lh_size, 0); } Uint erts_tot_link_lh_size(void) { - return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size); + return (Uint) erts_atomic_read_nob(&tot_link_lh_size); } void erts_destroy_monitor(ErtsMonitor *mon) @@ -245,7 +245,7 @@ void erts_destroy_monitor(ErtsMonitor *mon) erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon); } else { erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon); - erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); + erts_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint)); } } @@ -267,7 +267,7 @@ void erts_destroy_link(ErtsLink *lnk) erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk); } else { erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk); - erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); + erts_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint)); } } @@ -985,15 +985,14 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) DistEntry *dep; rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); if (!rp) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(p); + ERTS_ASSERT_IS_NOT_EXITING(p); if (is_atom(pid) && is_node_name_atom(pid) && (dep = erts_find_dist_entry(pid)) != NULL) { erts_printf("Dumping dist monitors-------------------\n"); - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); erts_dump_monitors(dep->monitors,0); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); erts_printf("Monitors dumped-------------------------\n"); - erts_deref_dist_entry(dep); BIF_RET(am_true); } else { BIF_ERROR(p,BADARG); @@ -1002,7 +1001,7 @@ Eterm erts_debug_dump_monitors_1(BIF_ALIST_1) erts_printf("Dumping pid monitors--------------------\n"); erts_dump_monitors(ERTS_P_MONITORS(rp),0); erts_printf("Monitors dumped-------------------------\n"); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); } } @@ -1030,15 +1029,14 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) } else { rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK); if (!rp) { - ERTS_SMP_ASSERT_IS_NOT_EXITING(p); + ERTS_ASSERT_IS_NOT_EXITING(p); if (is_atom(pid) && is_node_name_atom(pid) && (dep = erts_find_dist_entry(pid)) != NULL) { erts_printf("Dumping dist links----------------------\n"); - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); erts_dump_links(dep->nlinks,0); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); erts_printf("Links dumped----------------------------\n"); - erts_deref_dist_entry(dep); BIF_RET(am_true); } else { BIF_ERROR(p,BADARG); @@ -1048,7 +1046,7 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) erts_printf("Dumping pid links-----------------------\n"); erts_dump_links(ERTS_P_LINKS(rp), 0); erts_printf("Links dumped----------------------------\n"); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 2d70f0d874..d659842b7e 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -48,11 +48,7 @@ static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) static void erts_msacc_reset(ErtsMsAcc *msacc); static ErtsMsAcc* get_msacc(void); -#ifdef USE_THREADS erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); -#else -ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL; -#endif #ifndef ERTS_MSACC_ALWAYS_ON int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); #endif @@ -60,10 +56,8 @@ int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); static Eterm *erts_msacc_state_atoms = NULL; static erts_rwmtx_t msacc_mutex; static ErtsMsAcc *msacc_managed = NULL; -#ifdef USE_THREADS static ErtsMsAcc *msacc_unmanaged = NULL; static Uint msacc_unmanaged_count = 0; -#endif #if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT #define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + ERTS_REF_THING_SIZE) @@ -76,12 +70,9 @@ void erts_msacc_early_init(void) { #ifndef ERTS_MSACC_ALWAYS_ON erts_msacc_enabled = 0; #endif - erts_rwmtx_init(&msacc_mutex,"msacc_list_mutex"); -#ifdef USE_THREADS + erts_rwmtx_init(&msacc_mutex, "msacc_list_mutex", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); erts_tsd_key_create(&erts_msacc_key,"erts_msacc_key"); -#else - erts_msacc = NULL; -#endif } void erts_msacc_init(void) { @@ -106,10 +97,10 @@ void erts_msacc_init_thread(char *type, int id, int managed) { msacc->tid = erts_thr_self(); msacc->perf_counter = 0; -#ifdef USE_THREADS erts_rwmtx_rwlock(&msacc_mutex); if (!managed) { - erts_mtx_init(&msacc->mtx,"msacc_unmanaged_mutex"); + erts_mtx_init(&msacc->mtx, "msacc_unmanaged_mutex", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); msacc->next = msacc_unmanaged; msacc_unmanaged = msacc; msacc_unmanaged_count++; @@ -119,9 +110,6 @@ void erts_msacc_init_thread(char *type, int id, int managed) { msacc_managed = msacc; } erts_rwmtx_rwunlock(&msacc_mutex); -#else - msacc_managed = msacc; -#endif erts_msacc_reset(msacc); @@ -214,7 +202,7 @@ typedef struct { Eterm ref; Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; } ErtsMSAccReq; static ErtsMsAcc* get_msacc(void) { @@ -265,7 +253,7 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } @@ -301,7 +289,7 @@ reply_msacc(void *vmsaccrp) erts_proc_dec_refc(msaccrp->proc); - if (erts_smp_atomic32_dec_read_nob(&msaccrp->refc) == 0) + if (erts_atomic32_dec_read_nob(&msaccrp->refc) == 0) erts_free(ERTS_ALC_T_MSACC, vmsaccrp); } @@ -368,14 +356,10 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) msaccrp->ref = STORE_NC(&hp, NULL, ref); msaccrp->req_sched = esdp->no; -#ifdef ERTS_SMP *threads = erts_no_schedulers; *threads += 1; /* aux thread */ -#else - *threads = 1; -#endif - erts_smp_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads); + erts_atomic32_init_nob(&msaccrp->refc,(erts_aint32_t)*threads); erts_proc_add_refc(c_p, *threads); @@ -384,12 +368,9 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) erts_no_schedulers, reply_msacc, (void *) msaccrp); -#ifdef ERTS_SMP /* aux thread */ erts_schedule_misc_aux_work(0, reply_msacc, (void *) msaccrp); -#endif -#ifdef USE_THREADS /* Manage unmanaged threads */ switch (action) { case ERTS_MSACC_GATHER: { @@ -466,7 +447,6 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) default: { ASSERT(0); } } -#endif *threads = make_small(*threads); diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index d64ef8c8b9..2d4637f800 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -159,11 +159,7 @@ struct erl_msacc_t_ { #ifdef ERTS_ENABLE_MSACC -#ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; -#else -extern ErtsMsAcc *erts_msacc; -#endif #ifdef ERTS_MSACC_ALWAYS_ON #define erts_msacc_enabled 1 @@ -171,13 +167,8 @@ extern ErtsMsAcc *erts_msacc; extern int erts_msacc_enabled; #endif -#ifdef USE_THREADS #define ERTS_MSACC_TSD_GET() erts_tsd_get(erts_msacc_key) #define ERTS_MSACC_TSD_SET(tsd) erts_tsd_set(erts_msacc_key,tsd) -#else -#define ERTS_MSACC_TSD_GET() erts_msacc -#define ERTS_MSACC_TSD_SET(tsd) erts_msacc = tsd -#endif void erts_msacc_early_init(void); void erts_msacc_init(void); @@ -327,8 +318,8 @@ ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { if (ERTS_UNLIKELY(msacc->unmanaged)) { erts_mtx_lock(&msacc->mtx); - msacc->state = new_state; if (ERTS_LIKELY(!msacc->perf_counter)) { + msacc->state = new_state; erts_mtx_unlock(&msacc->mtx); return; } diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 19bb7d5b31..f2a660f085 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -583,8 +583,10 @@ void erts_mtrace_init(char *receiver, char *nodename) byte ip_addr[4]; Uint16 port; - erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); + erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); + erts_mtx_init(&mtrace_op_mutex, "mtrace_op", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c index 1bebc1eda4..f97e86bf95 100644 --- a/erts/emulator/beam/erl_nfunc_sched.c +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -113,7 +113,7 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, NifExport* nep; int i; - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); if (dirty_shadow_proc) { diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 55a3a6dbf6..b8a4e4ebc3 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -144,9 +144,9 @@ ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); - ERTS_SMP_LC_ASSERT(!(c_p->static_flags + ERTS_LC_ASSERT(!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); c_p->current = ep->current; @@ -193,7 +193,6 @@ erts_nif_export_check_save_trace(Process *c_p, Eterm result, ERTS_GLB_INLINE Process * erts_proc_shadow2real(Process *c_p) { -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) { Process *real_c_p = c_p->next; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); @@ -201,7 +200,6 @@ erts_proc_shadow2real(Process *c_p) return real_c_p; } ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); -#endif return c_p; } @@ -213,11 +211,10 @@ erts_proc_shadow2real(Process *c_p) #define ERTS_NFUNC_SCHED_INTERNALS__ #define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ - (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I)) \ - || BeamOp(op_call_nif) == (BeamInstr *) (*(I))), \ + (ASSERT(BeamIsOpCode(*(I), op_apply_bif) || \ + BeamIsOpCode(*(I), op_call_nif)), \ ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) -#ifdef ERTS_DIRTY_SCHEDULERS #include "erl_message.h" #include <stddef.h> @@ -235,7 +232,7 @@ erts_flush_dirty_shadow_proc(Process *sproc) Process *c_p = sproc->next; ASSERT(sproc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); ASSERT(c_p->stop == sproc->stop); @@ -283,7 +280,7 @@ erts_cache_dirty_shadow_proc(Process *sproc) Process *c_p = sproc->next; ASSERT(c_p); ASSERT(sproc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); sproc->htop = c_p->htop; @@ -311,7 +308,7 @@ erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) sproc = esdp->dirty_shadow_process; ASSERT(sproc); ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + ASSERT(erts_atomic32_read_nob(&sproc->state) == (ERTS_PSFLG_ACTIVE | ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_PROXY)); @@ -326,7 +323,6 @@ erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* ERTS_DIRTY_SCHEDULERS */ #endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 4815e5e7bb..f7f12efe28 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -56,6 +56,7 @@ #include "erl_process.h" #include "erl_bif_unique.h" #include "erl_utils.h" +#include "erl_io_queue.h" #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -66,7 +67,6 @@ #include <limits.h> #include <stddef.h> /* offsetof */ - /* Information about a loaded nif library. * Each successful call to erlang:load_nif will allocate an instance of * erl_module_nif. Two calls opening the same library will thus have the same @@ -138,7 +138,7 @@ execution_state(ErlNifEnv *env, Process **c_pp, int *schedp) Process *c_p = env->proc; if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) { - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); } else { @@ -220,7 +220,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, ASSERT(esdp); if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_atomic32_read_nob(&p->state); ASSERT(p->scheduler_data == esdp); ASSERT((state & (ERTS_PSFLG_RUNNING @@ -287,12 +287,12 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, else dirty_shadow_proc = env->proc; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, c_p->current, c_p->cp, - (BeamInstr) em_call_nif, + BeamOpCodeAddr(op_call_nif), direct_fp, indirect_fp, mod, func_name, argc, (const Eterm *) argv); @@ -304,7 +304,6 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, return (ERL_NIF_TERM) THE_NON_VALUE; } -#ifdef ERTS_DIRTY_SCHEDULERS static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); @@ -320,7 +319,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * ErlNifEnv env; ERL_NIF_TERM result; #ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state); + erts_aint32_t state = erts_atomic32_read_nob(&c_p->state); ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); @@ -343,14 +342,14 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); - erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); ASSERT(env.proc->next == c_p); @@ -394,24 +393,19 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * return exiting; } -#endif static void full_flush_env(ErlNifEnv* env) { flush_env(env); -#ifdef ERTS_DIRTY_SCHEDULERS if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) /* Dirty nif call using shadow process struct */ erts_flush_dirty_shadow_proc(env->proc); -#endif } static void full_cache_env(ErlNifEnv* env) { -#ifdef ERTS_DIRTY_SCHEDULERS if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) erts_cache_dirty_shadow_proc(env->proc); -#endif cache_env(env); } @@ -564,7 +558,6 @@ void enif_clear_env(ErlNifEnv* env) free_tmp_objs(env); } -#ifdef ERTS_SMP #ifdef DEBUG static int enif_send_delay = 0; #define ERTS_FORCE_ENIF_SEND_DELAY() (enif_send_delay++ % 2 == 0) @@ -588,7 +581,11 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) ErlTraceMessageQueue *msgq, **last_msgq; int reds = 0; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); + /* Only one thread at a time is allowed to flush trace messages, + so we require the main lock to be held when doing the flush */ + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); + + erts_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); msgq = c_p->trace_msg_q; @@ -607,7 +604,7 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) msgq->first = NULL; msgq->last = &msgq->first; msgq->len = 0; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); ASSERT(len != 0); @@ -620,13 +617,13 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) if (rp->common.id == c_p->common.id) rp_locks &= ~c_p_locks; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); reds += len; } else { erts_cleanup_messages(first); } reds += 1; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); + erts_proc_lock(c_p, ERTS_PROC_LOCK_TRACE); msgq = msgq->next; } while (msgq); @@ -643,21 +640,18 @@ int erts_flush_trace_messages(Process *c_p, ErtsProcLocks c_p_locks) } error: - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_TRACE); return reds; } -#endif int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; ErtsProcLocks rp_locks = 0; -#ifdef ERTS_SMP ErtsProcLocks lc_locks = 0; -#endif Process* rp; Process* c_p; ErtsMessage *mp; @@ -666,13 +660,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, execution_state(env, &c_p, &scheduler); -#ifndef ERTS_SMP - if (!scheduler) { - erts_exit(ERTS_ABORT_EXIT, - "enif_send: called from non-scheduler thread on non-SMP VM"); - return 0; - } -#endif if (scheduler > 0) { /* Normal scheduler */ rp = erts_proc_lookup(receiver); @@ -686,7 +673,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, return 0; if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } } @@ -695,7 +682,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ERTS_P2P_FLG_INC_REFC); if (!rp) { if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); return 0; } } @@ -730,7 +717,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, full_cache_env(env); } else { - erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state); + erts_aint_t state = erts_atomic32_read_nob(&rp->state); if (state & ERTS_PSFLG_OFF_HEAP_MSGQ) { mp = erts_alloc_message(sz, &hp); ohp = sz == 0 ? NULL : &mp->hfrag.off_heap; @@ -756,7 +743,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, full_cache_env(env); } } -#ifdef ERTS_SMP else { /* This clause is taken when the nif is called in the context of a traced process. We do not know which locks we have @@ -767,7 +753,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process *t_p = env->tracee; - erts_smp_proc_lock(t_p, ERTS_PROC_LOCK_TRACE); + erts_proc_lock(t_p, ERTS_PROC_LOCK_TRACE); msgq = t_p->trace_msg_q; @@ -784,7 +770,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #endif if (ERTS_FORCE_ENIF_SEND_DELAY() || msgq || rp_locks & ERTS_PROC_LOCK_MSGQ || - erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { + erts_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { if (!msgq) { msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, @@ -798,36 +784,33 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, msgq->next = t_p->trace_msg_q; t_p->trace_msg_q = msgq; - erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); erts_schedule_flush_trace_messages(t_p, 0); } else { msgq->len++; *msgq->last = mp; msgq->last = &mp->next; - erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); } goto done; } else { - erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); rp_locks &= ~ERTS_PROC_LOCK_TRACE; rp_locks |= ERTS_PROC_LOCK_MSGQ; } } -#endif /* ERTS_SMP */ erts_queue_message(rp, rp_locks, mp, msg, c_p ? c_p->common.id : am_undefined); -#ifdef ERTS_SMP done: if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) - erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); + erts_proc_unlock(rp, rp_locks & ~lc_locks); if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); -#endif + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); if (scheduler <= 0) erts_proc_dec_refc(rp); @@ -857,15 +840,9 @@ enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port, if (scheduler > 0) prt = erts_port_lookup(to_port->port_id, iflags); else { -#ifdef ERTS_SMP if (ERTS_PROC_IS_EXITING(c_p)) return 0; prt = erts_thr_port_lookup(to_port->port_id, iflags); -#else - erts_exit(ERTS_ABORT_EXIT, - "enif_port_command: called from non-scheduler " - "thread on non-SMP VM"); -#endif } if (!prt) @@ -889,26 +866,27 @@ static Eterm call_whereis(ErlNifEnv *env, Eterm name) Process *c_p; Eterm res; int scheduler; - int unlock; execution_state(env, &c_p, &scheduler); ASSERT((c_p && scheduler) || (!c_p && !scheduler)); - unlock = 0; if (scheduler < 0) { /* dirty scheduler */ if (ERTS_PROC_IS_EXITING(c_p)) return 0; - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); - unlock = 1; - } + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + c_p = NULL; /* as we don't have main lock */ } - res = erts_whereis_name_to_id(c_p, name); - if (unlock) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + if (c_p) { + /* main lock may be released below and c_p->htop updated by others */ + flush_env(env); + } + res = erts_whereis_name_to_id(c_p, name); + if (c_p) + cache_env(env); return res; } @@ -1847,18 +1825,11 @@ int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc) if (scheduler > 0) return !!erts_proc_lookup(proc->pid); else { -#ifdef ERTS_SMP Process* rp = erts_pid2proc_opt(NULL, 0, proc->pid, 0, ERTS_P2P_FLG_INC_REFC); if (rp) erts_proc_dec_refc(rp); return !!rp; -#else - erts_exit(ERTS_ABORT_EXIT, "enif_is_process_alive: " - "called from non-scheduler thread " - "in non-smp emulator"); - return 0; -#endif } } @@ -1874,17 +1845,10 @@ int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port) if (scheduler > 0) return !!erts_port_lookup(port->port_id, iflags); else { -#ifdef ERTS_SMP Port *prt = erts_thr_port_lookup(port->port_id, iflags); if (prt) erts_port_dec_refc(prt); return !!prt; -#else - erts_exit(ERTS_ABORT_EXIT, "enif_is_port_alive: " - "called from non-scheduler thread " - "in non-smp emulator"); - return 0; -#endif } } @@ -2092,7 +2056,7 @@ ErlNifResourceType* open_resource_type(ErlNifEnv* env, ErlNifResourceFlags op = flags; Eterm module_am, name_am; - ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(erts_thr_progress_is_blocking()); module_am = make_atom(env->mod_nif->mod->module); name_am = enif_make_atom(env, name_str); @@ -2231,19 +2195,14 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context) rp = erts_proc_lookup(mon->u.pid); } else { -#ifdef ERTS_SMP rp = erts_proc_lookup_inc_refc(mon->u.pid); -#else - ASSERT(!"nif monitor destruction in non-scheduler thread"); - rp = NULL; -#endif } if (!rp) { is_exiting = 1; } if (rp) { - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (ERTS_PROC_IS_EXITING(rp)) { is_exiting = 1; } else { @@ -2251,11 +2210,9 @@ static void destroy_one_monitor(ErtsMonitor* mon, void* context) ASSERT(rmon); is_exiting = 0; } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); -#ifdef ERTS_SMP + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (ctx->scheduler <= 0) erts_proc_dec_refc(rp); -#endif } if (is_exiting) { ctx->resource->monitors->pending_failed_fire++; @@ -2281,34 +2238,7 @@ static void destroy_all_monitors(ErtsMonitor* monitors, ErtsResource* resource) } -#ifdef ERTS_SMP # define NIF_RESOURCE_DTOR &nif_resource_dtor -#else -# define NIF_RESOURCE_DTOR &nosmp_nif_resource_dtor_prologue - -/* - * NO-SMP: Always run resource destructor on scheduler thread - * as we may have to remove process monitors. - */ -static int nif_resource_dtor(Binary*); - -static void nosmp_nif_resource_dtor_scheduled(void* vbin) -{ - erts_bin_free((Binary*)vbin); -} - -static int nosmp_nif_resource_dtor_prologue(Binary* bin) -{ - if (is_scheduler()) { - return nif_resource_dtor(bin); - } - else { - erts_schedule_misc_aux_work(1, nosmp_nif_resource_dtor_scheduled, bin); - return 0; /* do not free */ - } -} - -#endif /* !ERTS_SMP */ static int nif_resource_dtor(Binary* bin) { @@ -2320,7 +2250,7 @@ static int nif_resource_dtor(Binary* bin) ErtsResourceMonitors* rm = resource->monitors; ASSERT(type->down); - erts_smp_mtx_lock(&rm->lock); + erts_mtx_lock(&rm->lock); ASSERT(erts_refc_read(&bin->intern.refc, 0) == 0); if (rm->root) { ASSERT(!rm->is_dying); @@ -2342,11 +2272,11 @@ static int nif_resource_dtor(Binary* bin) */ ASSERT(!rm->is_dying); rm->is_dying = 1; - erts_smp_mtx_unlock(&rm->lock); + erts_mtx_unlock(&rm->lock); return 0; } - erts_smp_mtx_unlock(&rm->lock); - erts_smp_mtx_destroy(&rm->lock); + erts_mtx_unlock(&rm->lock); + erts_mtx_destroy(&rm->lock); } if (type->dtor != NULL) { @@ -2387,12 +2317,12 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) ASSERT(rmp); ASSERT(resource->type->down); - erts_smp_mtx_lock(&rmp->lock); + erts_mtx_lock(&rmp->lock); rmon = erts_remove_monitor(&rmp->root, ref); if (!rmon) { int free_me = (--rmp->pending_failed_fire == 0) && rmp->is_dying; ASSERT(rmp->pending_failed_fire >= 0); - erts_smp_mtx_unlock(&rmp->lock); + erts_mtx_unlock(&rmp->lock); if (free_me) { ASSERT(erts_refc_read(&bin->binary.intern.refc, 0) == 0); @@ -2408,10 +2338,10 @@ void erts_fire_nif_monitor(ErtsResource* resource, Eterm pid, Eterm ref) * we avoid calling 'down' and just silently remove the monitor. * This can happen even for non smp as destructor calls may be scheduled. */ - erts_smp_mtx_unlock(&rmp->lock); + erts_mtx_unlock(&rmp->lock); } else { - erts_smp_mtx_unlock(&rmp->lock); + erts_mtx_unlock(&rmp->lock); ASSERT(rmon->u.pid == pid); erts_ref_to_driver_monitor(ref, &nif_monitor); @@ -2456,7 +2386,8 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t data_sz) erts_refc_inc(&resource->type->refc, 2); if (type->down) { resource->monitors = (ErtsResourceMonitors*) (resource->data + monitors_offs); - erts_smp_mtx_init(&resource->monitors->lock, "resource_monitors"); + erts_mtx_init(&resource->monitors->lock, "resource_monitors", NIL, + ERTS_LOCK_FLAGS_CATEGORY_GENERIC); resource->monitors->root = NULL; resource->monitors->pending_failed_fire = 0; resource->monitors->is_dying = 0; @@ -2654,7 +2585,6 @@ nif_export_restore(Process *c_p, NifExport *ep, Eterm res) } -#ifdef ERTS_DIRTY_SCHEDULERS /* * Finalize a dirty NIF call. This function is scheduled to cause the VM to @@ -2724,7 +2654,7 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp, execution_state(env, &proc, NULL); - (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (void) erts_atomic32_read_bset_nob(&proc->state, (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC), (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND @@ -2762,7 +2692,7 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, ASSERT(is_atom(mod) && is_atom(func)); ASSERT(fp); - (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (void) erts_atomic32_read_bset_nob(&proc->state, (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC), dirty_psflg); @@ -2782,7 +2712,6 @@ static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } -#endif /* ERTS_DIRTY_SCHEDULERS */ /* * NIF execution wrapper used by enif_schedule_nif() for regular NIFs. It @@ -2856,24 +2785,20 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, if (scheduler <= 0) { if (scheduler == 0) enif_make_badarg(env); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } if (flags == 0) result = schedule(env, execute_nif, fp, proc->current->module, fun_name_atom, argc, argv); else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) { -#ifdef ERTS_DIRTY_SCHEDULERS result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); -#else - result = enif_raise_exception(env, am_notsup); -#endif } else result = enif_make_badarg(env); if (scheduler < 0) - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); return result; } @@ -2889,12 +2814,10 @@ enif_thread_type(void) switch (esdp->type) { case ERTS_SCHED_NORMAL: return ERL_NIF_THR_NORMAL_SCHEDULER; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: return ERL_NIF_THR_DIRTY_CPU_SCHEDULER; case ERTS_SCHED_DIRTY_IO: return ERL_NIF_THR_DIRTY_IO_SCHEDULER; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return -1; @@ -3215,27 +3138,19 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, execution_state(env, NULL, &scheduler); -#ifdef ERTS_SMP if (scheduler > 0) /* Normal scheduler */ rp = erts_proc_lookup_raw(target_pid->pid); else rp = erts_proc_lookup_raw_inc_refc(target_pid->pid); -#else - if (scheduler <= 0) { - erts_exit(ERTS_ABORT_EXIT, "enif_monitor_process: called from " - "non-scheduler thread on non-SMP VM"); - } - rp = erts_proc_lookup(target_pid->pid); -#endif if (!rp) return 1; ref = erts_make_ref_in_buffer(tmp); - erts_smp_mtx_lock(&rsrc->monitors->lock); - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); - if (ERTS_PSFLG_FREE & erts_smp_atomic32_read_nob(&rp->state)) { + erts_mtx_lock(&rsrc->monitors->lock); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); + if (ERTS_PSFLG_FREE & erts_atomic32_read_nob(&rp->state)) { retval = 1; } else { @@ -3243,13 +3158,11 @@ int enif_monitor_process(ErlNifEnv* env, void* obj, const ErlNifPid* target_pid, erts_add_monitor(&ERTS_P_MONITORS(rp), MON_NIF_TARGET, ref, (UWord)rsrc, NIL); retval = 0; } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); - erts_smp_mtx_unlock(&rsrc->monitors->lock); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_mtx_unlock(&rsrc->monitors->lock); -#ifdef ERTS_SMP if (scheduler <= 0) erts_proc_dec_refc(rp); -#endif if (monitor) erts_ref_to_driver_monitor(ref,monitor); @@ -3277,35 +3190,27 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit ref = erts_driver_monitor_to_ref(ref_heap, monitor); - erts_smp_mtx_lock(&rsrc->monitors->lock); + erts_mtx_lock(&rsrc->monitors->lock); mon = erts_remove_monitor(&rsrc->monitors->root, ref); if (mon == NULL) { - erts_smp_mtx_unlock(&rsrc->monitors->lock); + erts_mtx_unlock(&rsrc->monitors->lock); return 1; } ASSERT(mon->type == MON_ORIGIN); ASSERT(is_internal_pid(mon->u.pid)); -#ifdef ERTS_SMP if (scheduler > 0) /* Normal scheduler */ rp = erts_proc_lookup(mon->u.pid); else rp = erts_proc_lookup_inc_refc(mon->u.pid); -#else - if (scheduler <= 0) { - erts_exit(ERTS_ABORT_EXIT, "enif_demonitor_process: called from " - "non-scheduler thread on non-SMP VM"); - } - rp = erts_proc_lookup(mon->u.pid); -#endif if (!rp) { is_exiting = 1; } else { - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (ERTS_PROC_IS_EXITING(rp)) { is_exiting = 1; } else { @@ -3313,17 +3218,15 @@ int enif_demonitor_process(ErlNifEnv* env, void* obj, const ErlNifMonitor* monit ASSERT(rmon); is_exiting = 0; } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); -#ifdef ERTS_SMP if (scheduler <= 0) erts_proc_dec_refc(rp); -#endif } if (is_exiting) { rsrc->monitors->pending_failed_fire++; } - erts_smp_mtx_unlock(&rsrc->monitors->lock); + erts_mtx_unlock(&rsrc->monitors->lock); if (rmon) { ASSERT(rmon->type == MON_NIF_TARGET); @@ -3342,6 +3245,363 @@ int enif_compare_monitors(const ErlNifMonitor *monitor1, ERTS_REF_THING_SIZE*sizeof(Eterm)); } +ErlNifIOQueue *enif_ioq_create(ErlNifIOQueueOpts opts) +{ + ErlNifIOQueue *q; + + if (opts != ERL_NIF_IOQ_NORMAL) + return NULL; + + q = enif_alloc(sizeof(ErlNifIOQueue)); + if (!q) return NULL; + erts_ioq_init(q, ERTS_ALC_T_NIF, 0); + + return q; +} + +void enif_ioq_destroy(ErlNifIOQueue *q) +{ + erts_ioq_clear(q); + enif_free(q); +} + +/* If the iovec was preallocated (Stack or otherwise) it needs to be marked as + * such to perform a proper free. */ +#define ERL_NIF_IOVEC_FLAGS_PREALLOC (1 << 0) + +void enif_free_iovec(ErlNifIOVec *iov) +{ + int i; + /* Decrement the refc of all the binaries */ + for (i = 0; i < iov->iovcnt; i++) { + Binary *bptr = ((Binary**)iov->ref_bins)[i]; + /* bptr can be null if enq_binary was used */ + if (bptr && erts_refc_dectest(&bptr->intern.refc, 0) == 0) { + erts_bin_free(bptr); + } + } + + if (!(iov->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { + enif_free(iov); + } +} + +typedef struct { + UWord sublist_length; + Eterm sublist_start; + Eterm sublist_end; + + UWord offheap_size; + UWord onheap_size; + + UWord iovec_len; +} iovec_slice_t; + +static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *result) { + Eterm lookahead; + + result->sublist_start = list; + result->sublist_length = 0; + result->offheap_size = 0; + result->onheap_size = 0; + result->iovec_len = 0; + + lookahead = result->sublist_start; + + while (is_list(lookahead)) { + Eterm *binary_header, binary; + Eterm *cell; + UWord size; + + cell = list_val(lookahead); + binary = CAR(cell); + + if (!is_binary(binary)) { + return 0; + } + + size = binary_size(binary); + binary_header = binary_val(binary); + + /* If we're a sub-binary we'll need to check our underlying binary to + * determine whether we're on-heap or not. */ + if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { + ErlSubBin *sb = (ErlSubBin*)binary_header; + + /* Reject bitstrings */ + if((sb->bitoffs + sb->bitsize) > 0) { + return 0; + } + + ASSERT(size <= binary_size(sb->orig)); + binary_header = binary_val(sb->orig); + } + + if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { + ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + + result->iovec_len += 1; + result->onheap_size += size; + } else { + ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); + + result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; + result->offheap_size += size; + } + + result->sublist_length += 1; + lookahead = CDR(cell); + + if(result->sublist_length >= max_length) { + break; + } + } + + if (!is_nil(lookahead) && !is_list(lookahead)) { + return 0; + } + + result->sublist_end = lookahead; + + return 1; +} + +static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { + Eterm *parent_header; + Eterm parent_binary; + + int bit_offset, bit_size; + Uint byte_offset; + + ASSERT(is_binary(binary)); + + ERTS_GET_REAL_BIN(binary, parent_binary, byte_offset, bit_offset, bit_size); + + parent_header = binary_val(parent_binary); + + result->size = binary_size(binary); + result->bin_term = binary; + + if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { + ProcBin *pb = (ProcBin*)parent_header; + + ASSERT(pb->val != NULL); + ASSERT(byte_offset < pb->size); + ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes); + + result->data = (unsigned char*)&pb->bytes[byte_offset]; + result->ref_bin = (void*)pb->val; + } else { + ErlHeapBin *hb = (ErlHeapBin*)parent_header; + + ASSERT(thing_subtag(*parent_header) == HEAP_BINARY_SUBTAG); + + result->data = &((unsigned char*)&hb->data)[byte_offset]; + result->ref_bin = NULL; + } +} + +static int fill_iovec_with_slice(ErlNifEnv *env, + iovec_slice_t *slice, + ErlNifIOVec *iovec) { + UWord onheap_offset, iovec_idx; + ErlNifBinary onheap_data; + Eterm sublist_iterator; + + /* Set up a common refc binary for all on-heap binaries. */ + if (slice->onheap_size > 0) { + if (!enif_alloc_binary(slice->onheap_size, &onheap_data)) { + return 0; + } + } + + sublist_iterator = slice->sublist_start; + onheap_offset = 0; + iovec_idx = 0; + + while (sublist_iterator != slice->sublist_end) { + ErlNifBinary raw_data; + Eterm *cell; + + cell = list_val(sublist_iterator); + inspect_raw_binary_data(CAR(cell), &raw_data); + + /* If this isn't a refc binary, copy its contents to the onheap buffer + * and reference that instead. */ + if (raw_data.ref_bin == NULL) { + ASSERT(onheap_offset < onheap_data.size); + ASSERT(slice->onheap_size > 0); + + sys_memcpy(&onheap_data.data[onheap_offset], + raw_data.data, raw_data.size); + + raw_data.data = &onheap_data.data[onheap_offset]; + raw_data.ref_bin = onheap_data.ref_bin; + } + + ASSERT(raw_data.ref_bin != NULL); + + while (raw_data.size > 0) { + UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); + + ASSERT(iovec_idx < iovec->iovcnt); + + iovec->iov[iovec_idx].iov_base = raw_data.data; + iovec->iov[iovec_idx].iov_len = chunk_len; + + iovec->ref_bins[iovec_idx] = raw_data.ref_bin; + + raw_data.data += chunk_len; + raw_data.size -= chunk_len; + + iovec_idx += 1; + } + + sublist_iterator = CDR(cell); + } + + ASSERT(iovec_idx == iovec->iovcnt); + + if (env == NULL) { + int i; + for (i = 0; i < iovec->iovcnt; i++) { + Binary *refc_binary = (Binary*)(iovec->ref_bins[i]); + erts_refc_inc(&refc_binary->intern.refc, 1); + } + + if (slice->onheap_size > 0) { + /* Transfer ownership to the iovec; we've taken references to it in + * the above loop. */ + enif_release_binary(&onheap_data); + } + } else { + if (slice->onheap_size > 0) { + /* Attach the binary to our environment and let the GC take care of + * it after returning. */ + enif_make_binary(env, &onheap_data); + } + } + + return 1; +} + +static int create_iovec_from_slice(ErlNifEnv *env, + iovec_slice_t *slice, + ErlNifIOVec **result) { + ErlNifIOVec *iovec = *result; + + if (iovec && slice->iovec_len < ERL_NIF_IOVEC_SIZE) { + iovec->iov = iovec->small_iov; + iovec->ref_bins = iovec->small_ref_bin; + iovec->flags = ERL_NIF_IOVEC_FLAGS_PREALLOC; + } else { + UWord iov_offset, binv_offset, alloc_size; + char *alloc_base; + + iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlNifIOVec)); + binv_offset = iov_offset; + binv_offset += ERTS_ALC_DATA_ALIGN_SIZE(slice->iovec_len * sizeof(SysIOVec)); + alloc_size = binv_offset; + alloc_size += slice->iovec_len * sizeof(Binary*); + + /* If we have an environment we'll attach the allocated data to it. The + * GC will take care of releasing it later on. */ + if (env != NULL) { + ErlNifBinary gc_bin; + + if (!enif_alloc_binary(alloc_size, &gc_bin)) { + return 0; + } + + alloc_base = (char*)gc_bin.data; + enif_make_binary(env, &gc_bin); + } else { + alloc_base = enif_alloc(alloc_size); + } + + iovec = (ErlNifIOVec*)alloc_base; + iovec->iov = (SysIOVec*)(alloc_base + iov_offset); + iovec->ref_bins = (void**)(alloc_base + binv_offset); + iovec->flags = 0; + } + + iovec->size = slice->offheap_size + slice->onheap_size; + iovec->iovcnt = slice->iovec_len; + + if(!fill_iovec_with_slice(env, slice, iovec)) { + if (env == NULL && !(iovec->flags & ERL_NIF_IOVEC_FLAGS_PREALLOC)) { + enif_free(iovec); + } + + return 0; + } + + *result = iovec; + + return 1; +} + +int enif_inspect_iovec(ErlNifEnv *env, size_t max_elements, + ERL_NIF_TERM list, ERL_NIF_TERM *tail, + ErlNifIOVec **iov) { + iovec_slice_t slice; + + if(!examine_iovec_term(list, max_elements, &slice)) { + return 0; + } else if(!create_iovec_from_slice(env, &slice, iov)) { + return 0; + } + + (*tail) = slice.sublist_end; + + return 1; +} + +/* */ +int enif_ioq_enqv(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip) +{ + if(skip <= iov->size) { + return !erts_ioq_enqv(q, (ErtsIOVec*)iov, skip); + } + + return 0; +} + +int enif_ioq_enq_binary(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip) +{ + ErlNifIOVec vec = {1, bin->size, NULL, NULL, ERL_NIF_IOVEC_FLAGS_PREALLOC }; + Binary *ref_bin = (Binary*)bin->ref_bin; + int res; + vec.iov = vec.small_iov; + vec.ref_bins = vec.small_ref_bin; + vec.iov[0].iov_base = bin->data; + vec.iov[0].iov_len = bin->size; + ((Binary**)(vec.ref_bins))[0] = ref_bin; + + res = enif_ioq_enqv(q, &vec, skip); + enif_release_binary(bin); + return res; +} + +size_t enif_ioq_size(ErlNifIOQueue *q) +{ + return erts_ioq_size(q); +} + +int enif_ioq_deq(ErlNifIOQueue *q, size_t elems, size_t *size) +{ + if (erts_ioq_deq(q, elems) == -1) + return 0; + if (size) + *size = erts_ioq_size(q); + return 1; +} + +SysIOVec *enif_ioq_peek(ErlNifIOQueue *q, int *iovlen) +{ + return erts_ioq_peekq(q, iovlen); +} + /*************************************************************************** ** load_nif/2 ** ***************************************************************************/ @@ -3353,7 +3613,7 @@ static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsign int j; for (j = 0; j < n; ++j) { ErtsCodeInfo* ci = mod_code->functions[j]; - ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (f_atom == ci->mfa.function && arity == ci->mfa.arity) { return mod_code->functions+j; @@ -3549,8 +3809,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } /* Block system (is this the right place to do it?) */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); /* Find calling module */ ASSERT(BIF_P->current != NULL); @@ -3655,14 +3915,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) * dirty scheduler support, treat a non-zero flags field as * a load error. */ -#ifdef ERTS_DIRTY_SCHEDULERS if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", f->flags, mod_atom, f->name, f->arity); -#else - ret = load_nif_error(BIF_P, bad_lib, "NIF %T:%s/%u requires a runtime with dirty scheduler support.", - mod_atom, f->name, f->arity); -#endif } else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < BEAM_NIF_MIN_FUNC_SZ) @@ -3727,15 +3982,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) code_ptr = erts_codeinfo_to_code(ci); if (ci->u.gen_bp == NULL) { - code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); + code_ptr[0] = BeamOpCodeAddr(op_call_nif); } else { /* Function traced, patch the original instruction word */ GenericBp* g = ci->u.gen_bp; - ASSERT(code_ptr[0] == - (BeamInstr) BeamOp(op_i_generic_breakpoint)); - g->orig_instr = (BeamInstr) BeamOp(op_call_nif); + ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)); + g->orig_instr = BeamOpCodeAddr(op_call_nif); } -#ifdef ERTS_DIRTY_SCHEDULERS if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? @@ -3743,7 +3996,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (BeamInstr) static_schedule_dirty_cpu_nif; } else -#endif code_ptr[1] = (BeamInstr) f->fptr; code_ptr[2] = (BeamInstr) lib; } @@ -3761,8 +4013,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_sys_ddll_free_error(&errdesc); } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); @@ -3775,7 +4027,7 @@ erts_unload_nif(struct erl_module_nif* lib) { ErlNifResourceType* rt; ErlNifResourceType* next; - ASSERT(erts_smp_thr_progress_is_blocking()); + ASSERT(erts_thr_progress_is_blocking()); ASSERT(lib != NULL); ASSERT(lib->mod != NULL); @@ -3847,8 +4099,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, break; ASSERT(i < mod->entry.num_of_funcs); if (p) - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN - || erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN + || erts_thr_progress_is_blocking()); #endif if (p) { /* This is almost a normal nif call like in beam_emu, diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index b0d5c39798..d195721054 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -50,6 +50,7 @@ ** 2.9: 18.2 enif_getenv ** 2.10: Time API ** 2.11: 19.0 enif_snprintf +** 2.12: 20.0 add enif_queue */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 12 @@ -241,6 +242,28 @@ typedef enum { ERL_NIF_PHASH2 = 2 } ErlNifHash; +#define ERL_NIF_IOVEC_SIZE 16 + +typedef struct erl_nif_io_vec { + int iovcnt; /* length of vectors */ + size_t size; /* total size in bytes */ + SysIOVec *iov; + + /* internals (avert your eyes) */ + void **ref_bins; /* Binary[] */ + int flags; + + /* Used when stack allocating the io vec */ + SysIOVec small_iov[ERL_NIF_IOVEC_SIZE]; + void *small_ref_bin[ERL_NIF_IOVEC_SIZE]; +} ErlNifIOVec; + +typedef struct erts_io_queue ErlNifIOQueue; + +typedef enum { + ERL_NIF_IOQ_NORMAL = 1 +} ErlNifIOQueueOpts; + /* * Return values from enif_thread_type(). Negative values * reserved for specific types of non-scheduler threads. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 94c04cd126..9e573307d8 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -184,6 +184,21 @@ ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term ERL_NIF_API_FUNC_DECL(int, enif_whereis_pid, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPid *pid)); ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name, ErlNifPort *port)); +ERL_NIF_API_FUNC_DECL(ErlNifIOQueue *,enif_ioq_create,(ErlNifIOQueueOpts opts)); +ERL_NIF_API_FUNC_DECL(void,enif_ioq_destroy,(ErlNifIOQueue *q)); + +ERL_NIF_API_FUNC_DECL(int,enif_ioq_enq_binary,(ErlNifIOQueue *q, ErlNifBinary *bin, size_t skip)); +ERL_NIF_API_FUNC_DECL(int,enif_ioq_enqv,(ErlNifIOQueue *q, ErlNifIOVec *iov, size_t skip)); + +ERL_NIF_API_FUNC_DECL(size_t,enif_ioq_size,(ErlNifIOQueue *q)); +ERL_NIF_API_FUNC_DECL(int,enif_ioq_deq,(ErlNifIOQueue *q, size_t count, size_t *size)); + +ERL_NIF_API_FUNC_DECL(SysIOVec*,enif_ioq_peek,(ErlNifIOQueue *q, int *iovlen)); + +ERL_NIF_API_FUNC_DECL(int,enif_inspect_iovec,(ErlNifEnv *env, size_t max_length, ERL_NIF_TERM iovec_term, ERL_NIF_TERM *tail, ErlNifIOVec **iovec)); +ERL_NIF_API_FUNC_DECL(void,enif_free_iovec,(ErlNifIOVec *iov)); + + /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ @@ -348,6 +363,16 @@ ERL_NIF_API_FUNC_DECL(int, enif_whereis_port, (ErlNifEnv *env, ERL_NIF_TERM name # define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) # define enif_whereis_pid ERL_NIF_API_FUNC_MACRO(enif_whereis_pid) # define enif_whereis_port ERL_NIF_API_FUNC_MACRO(enif_whereis_port) +# define enif_ioq_create ERL_NIF_API_FUNC_MACRO(enif_ioq_create) +# define enif_ioq_destroy ERL_NIF_API_FUNC_MACRO(enif_ioq_destroy) +# define enif_ioq_enq ERL_NIF_API_FUNC_MACRO(enif_ioq_enq) +# define enif_ioq_enq_binary ERL_NIF_API_FUNC_MACRO(enif_ioq_enq_binary) +# define enif_ioq_enqv ERL_NIF_API_FUNC_MACRO(enif_ioq_enqv) +# define enif_ioq_size ERL_NIF_API_FUNC_MACRO(enif_ioq_size) +# define enif_ioq_deq ERL_NIF_API_FUNC_MACRO(enif_ioq_deq) +# define enif_ioq_peek ERL_NIF_API_FUNC_MACRO(enif_ioq_peek) +# define enif_inspect_iovec ERL_NIF_API_FUNC_MACRO(enif_inspect_iovec) +# define enif_free_iovec ERL_NIF_API_FUNC_MACRO(enif_free_iovec) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 3c5945d48d..0f3dfa797c 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -29,11 +29,13 @@ #include "error.h" #include "erl_thr_progress.h" #include "dtrace-wrapper.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" Hash erts_dist_table; Hash erts_node_table; -erts_smp_rwmtx_t erts_dist_table_rwmtx; -erts_smp_rwmtx_t erts_node_table_rwmtx; +erts_rwmtx_t erts_dist_table_rwmtx; +erts_rwmtx_t erts_node_table_rwmtx; DistEntry *erts_hidden_dist_entries; DistEntry *erts_visible_dist_entries; @@ -57,6 +59,58 @@ static ErtsMonotonicTime node_tab_delete_delay; /* -- The distribution table ---------------------------------------------- */ +#define ErtsBin2DistEntry(B) \ + ((DistEntry *) ERTS_MAGIC_BIN_DATA((B))) +#define ErtsDistEntry2Bin(DEP) \ + ((Binary *) ERTS_MAGIC_BIN_FROM_DATA((DEP))) + +static ERTS_INLINE erts_aint_t +de_refc_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE erts_aint_t +de_refc_inc_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_inctest(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE void +de_refc_inc(DistEntry *dep, erts_aint_t min) +{ + erts_refc_inc(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +static ERTS_INLINE void +de_refc_dec(DistEntry *dep, erts_aint_t min) +{ +#ifdef DEBUG + (void) erts_refc_read(&ErtsDistEntry2Bin(dep)->intern.refc, min+1); +#endif + erts_bin_release(ErtsDistEntry2Bin(dep)); +} + +static ERTS_INLINE erts_aint_t +de_refc_dec_read(DistEntry *dep, erts_aint_t min) +{ + return erts_refc_dectest(&ErtsDistEntry2Bin(dep)->intern.refc, min); +} + +void +erts_ref_dist_entry(DistEntry *dep) +{ + ASSERT(dep); + de_refc_inc(dep, 1); +} + +void +erts_deref_dist_entry(DistEntry *dep) +{ + ASSERT(dep); + de_refc_dec(dep, 0); +} + #ifdef DEBUG static int is_in_de_list(DistEntry *dep, DistEntry *dep_list) @@ -85,44 +139,66 @@ dist_table_cmp(void *dep1, void *dep2) static void* dist_table_alloc(void *dep_tmpl) { - Eterm chnl_nr; +#ifdef DEBUG + erts_aint_t refc; +#endif Eterm sysname; + Binary *bin; DistEntry *dep; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; sysname = ((DistEntry *) dep_tmpl)->sysname; - chnl_nr = make_small((Uint) atom_val(sysname)); - dep = (DistEntry *) erts_alloc(ERTS_ALC_T_DIST_ENTRY, sizeof(DistEntry)); + + bin = erts_create_magic_binary_x(sizeof(DistEntry), + erts_dist_entry_destructor, + ERTS_ALC_T_DIST_ENTRY, + 0); + dep = ErtsBin2DistEntry(bin); dist_entries++; +#ifdef DEBUG + refc = +#else + (void) +#endif + de_refc_dec_read(dep, -1); + ASSERT(refc == -1); + dep->prev = NULL; - erts_smp_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); + erts_rwmtx_init_opt(&dep->rwmtx, &rwmtx_opt, "dist_entry", sysname, + ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); dep->sysname = sysname; dep->cid = NIL; + erts_atomic_init_nob(&dep->input_handler, (erts_aint_t) NIL); dep->connection_id = 0; dep->status = 0; dep->flags = 0; dep->version = 0; - erts_smp_mtx_init_x(&dep->lnk_mtx, "dist_entry_links", chnl_nr); + erts_mtx_init(&dep->lnk_mtx, "dist_entry_links", sysname, + ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); dep->node_links = NULL; dep->nlinks = NULL; dep->monitors = NULL; - erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); - dep->qflgs = 0; - dep->qsize = 0; + erts_mtx_init(&dep->qlock, "dist_entry_out_queue", sysname, + ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + erts_atomic32_init_nob(&dep->qflgs, 0); + erts_atomic_init_nob(&dep->qsize, 0); + erts_atomic64_init_nob(&dep->in, 0); + erts_atomic64_init_nob(&dep->out, 0); dep->out_queue.first = NULL; dep->out_queue.last = NULL; dep->suspended = NULL; + dep->tmp_out_queue.first = NULL; + dep->tmp_out_queue.last = NULL; dep->finalized_out_queue.first = NULL; dep->finalized_out_queue.last = NULL; - erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0); + erts_atomic_init_nob(&dep->dist_cmd_scheduled, 0); erts_port_task_handle_init(&dep->dist_cmd); dep->send = NULL; dep->cache = NULL; @@ -173,14 +249,14 @@ dist_table_free(void *vdep) erts_no_of_not_connected_dist_entries--; ASSERT(!dep->cache); - erts_smp_rwmtx_destroy(&dep->rwmtx); - erts_smp_mtx_destroy(&dep->lnk_mtx); - erts_smp_mtx_destroy(&dep->qlock); + erts_rwmtx_destroy(&dep->rwmtx); + erts_mtx_destroy(&dep->lnk_mtx); + erts_mtx_destroy(&dep->qlock); #ifdef DEBUG sys_memset(vdep, 0x77, sizeof(DistEntry)); #endif - erts_free(ERTS_ALC_T_DIST_ENTRY, (void *) dep); + erts_bin_free(ErtsDistEntry2Bin(dep)); ASSERT(dist_entries > 0); dist_entries--; @@ -192,25 +268,58 @@ erts_dist_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); hash_info(to, to_arg, &erts_dist_table); if (lock) - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); +} + +static ERTS_INLINE DistEntry *find_dist_entry(Eterm sysname, + int inc_refc, + int connected_only) +{ + DistEntry *res; + DistEntry de; + de.sysname = sysname; + erts_rwmtx_rlock(&erts_dist_table_rwmtx); + res = hash_get(&erts_dist_table, (void *) &de); + if (res) { + if (connected_only && is_nil(res->cid)) + res = NULL; + else { + int pend_delete; + erts_aint_t refc; + if (inc_refc) { + refc = de_refc_inc_read(res, 1); + pend_delete = refc < 2; + } + else { + refc = de_refc_read(res, 0); + pend_delete = refc < 1; + } + if (pend_delete) /* Pending delete */ + de_refc_inc(res, 1); + } + } + erts_rwmtx_runlock(&erts_dist_table_rwmtx); + return res; } DistEntry * erts_channel_no_to_dist_entry(Uint cno) { + /* + * Does NOT increase reference count! + */ + /* * For this node (and previous incarnations of this node), * ERST_INTERNAL_CHANNEL_NO (will always be 0 I guess) is used as * channel no. For other nodes, the atom index of the atom corresponding * to the node name is used as channel no. */ - if(cno == ERST_INTERNAL_CHANNEL_NO) { - erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); + if (cno == ERST_INTERNAL_CHANNEL_NO) return erts_this_dist_entry; - } if((cno > MAX_ATOM_INDEX) || (cno >= atom_table_size()) @@ -219,83 +328,100 @@ erts_channel_no_to_dist_entry(Uint cno) /* cno is a valid atom index; find corresponding dist entry (if there is one) */ - return erts_find_dist_entry(make_atom(cno)); + return find_dist_entry(make_atom(cno), 0, 0); } - DistEntry * erts_sysname_to_connected_dist_entry(Eterm sysname) { - DistEntry de; - DistEntry *res_dep; - de.sysname = sysname; - - if(erts_this_dist_entry->sysname == sysname) { - erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); + /* + * Does NOT increase reference count! + */ + if(erts_this_dist_entry->sysname == sysname) return erts_this_dist_entry; - } - - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); - if (res_dep) { - erts_aint_t refc = erts_smp_refc_inctest(&res_dep->refc, 1); - if (refc < 2) /* Pending delete */ - erts_smp_refc_inc(&res_dep->refc, 1); - } - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); - if (res_dep) { - int deref; - erts_smp_rwmtx_rlock(&res_dep->rwmtx); - deref = is_nil(res_dep->cid); - erts_smp_rwmtx_runlock(&res_dep->rwmtx); - if (deref) { - erts_deref_dist_entry(res_dep); - res_dep = NULL; - } - } - return res_dep; + return find_dist_entry(sysname, 0, 1); } DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) { + /* + * This function DOES increase reference count! + */ DistEntry *res; DistEntry de; erts_aint_t refc; - res = erts_find_dist_entry(sysname); + res = find_dist_entry(sysname, 1, 0); if (res) return res; de.sysname = sysname; - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); res = hash_put(&erts_dist_table, (void *) &de); - refc = erts_smp_refc_inctest(&res->refc, 0); + refc = de_refc_inc_read(res, 0); if (refc < 2) /* New or pending delete */ - erts_smp_refc_inc(&res->refc, 1); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + de_refc_inc(res, 1); + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); return res; } DistEntry *erts_find_dist_entry(Eterm sysname) { - DistEntry *res; - DistEntry de; - de.sysname = sysname; - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); - res = hash_get(&erts_dist_table, (void *) &de); - if (res) { - erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 1); - if (refc < 2) /* Pending delete */ - erts_smp_refc_inc(&res->refc, 1); - } - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); - return res; + /* + * Does NOT increase reference count! + */ + return find_dist_entry(sysname, 0, 0); } -static void try_delete_dist_entry(void *vdep) +DistEntry * +erts_dhandle_to_dist_entry(Eterm dhandle) { - DistEntry *dep = (DistEntry *) vdep; + Binary *bin; + if (!is_internal_magic_ref(dhandle)) + return NULL; + bin = erts_magic_ref2bin(dhandle); + if (ERTS_MAGIC_BIN_DESTRUCTOR(bin) != erts_dist_entry_destructor) + return NULL; + return ErtsBin2DistEntry(bin); +} + +Eterm +erts_make_dhandle(Process *c_p, DistEntry *dep) +{ + Binary *bin; + Eterm *hp; + + bin = ErtsDistEntry2Bin(dep); + ASSERT(bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == erts_dist_entry_destructor); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &c_p->off_heap, bin); +} + +static void try_delete_dist_entry(void *vbin); + +static void +prepare_try_delete_dist_entry(void *vbin) +{ + Binary *bin = (Binary *) vbin; + DistEntry *dep = ErtsBin2DistEntry(bin); + Uint size; + erts_aint_t refc; + + refc = de_refc_read(dep, 0); + if (refc > 0) + return; + + size = ERTS_MAGIC_BIN_SIZE(sizeof(DistEntry)); + erts_schedule_thr_prgr_later_cleanup_op(try_delete_dist_entry, + vbin, &dep->later_op, size); +} + +static void try_delete_dist_entry(void *vbin) +{ + Binary *bin = (Binary *) vbin; + DistEntry *dep = ErtsBin2DistEntry(bin); erts_aint_t refc; - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); /* * Another thread might have looked up this dist entry after * we decided to delete it (refc became zero). If so, the other @@ -311,26 +437,39 @@ static void try_delete_dist_entry(void *vdep) * * If refc > 0, the entry is in use. Keep the entry. */ - refc = erts_smp_refc_dectest(&dep->refc, -1); + refc = de_refc_dec_read(dep, -1); if (refc == -1) (void) hash_erase(&erts_dist_table, (void *) dep); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); - - if (refc == 0) - erts_schedule_delete_dist_entry(dep); + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); + + if (refc == 0) { + if (node_tab_delete_delay == 0) + prepare_try_delete_dist_entry(vbin); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + prepare_try_delete_dist_entry, + vbin); + } } -void erts_schedule_delete_dist_entry(DistEntry *dep) +int erts_dist_entry_destructor(Binary *bin) { - ASSERT(dep != erts_this_dist_entry); - if (dep != erts_this_dist_entry) { - if (node_tab_delete_delay == 0) - try_delete_dist_entry((void *) dep); - else if (node_tab_delete_delay > 0) - erts_start_timer_callback(node_tab_delete_delay, - try_delete_dist_entry, - (void *) dep); - } + DistEntry *dep = ErtsBin2DistEntry(bin); + erts_aint_t refc; + + refc = de_refc_read(dep, -1); + + if (refc == -1) + return 1; /* Allow deallocation of structure... */ + + if (node_tab_delete_delay == 0) + prepare_try_delete_dist_entry((void *) bin); + else if (node_tab_delete_delay > 0) + erts_start_timer_callback(node_tab_delete_delay, + prepare_try_delete_dist_entry, + (void *) bin); + + return 0; } Uint @@ -345,7 +484,7 @@ erts_dist_table_size(void) int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); + erts_rwmtx_rlock(&erts_dist_table_rwmtx); #ifdef DEBUG hash_get_info(&hi, &erts_dist_table); ASSERT(dist_entries == hi.objs); @@ -372,18 +511,18 @@ erts_dist_table_size(void) + dist_entries*sizeof(DistEntry) + erts_dist_cache_size()); if (lock) - erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); return res; } void erts_set_dist_entry_not_connected(DistEntry *dep) { - ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); - ASSERT(is_internal_port(dep->cid)); + ASSERT(is_internal_port(dep->cid) || is_internal_pid(dep->cid)); if(dep->flags & DFLAG_PUBLISHED) { if(dep->prev) { @@ -427,18 +566,18 @@ erts_set_dist_entry_not_connected(DistEntry *dep) } erts_not_connected_dist_entries = dep; erts_no_of_not_connected_dist_entries++; - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); } void erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) { - ERTS_SMP_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + ERTS_LC_ASSERT(erts_lc_is_de_rwlocked(dep)); + erts_rwmtx_rwlock(&erts_dist_table_rwmtx); ASSERT(dep != erts_this_dist_entry); ASSERT(is_nil(dep->cid)); - ASSERT(is_internal_port(cid)); + ASSERT(is_internal_port(cid) || is_internal_pid(cid)); if(dep->prev) { ASSERT(is_in_de_list(dep, erts_not_connected_dist_entries)); @@ -458,10 +597,19 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) dep->status |= ERTS_DE_SFLG_CONNECTED; dep->flags = flags; dep->cid = cid; + erts_atomic_set_nob(&dep->input_handler, + (erts_aint_t) cid); + dep->connection_id++; dep->connection_id &= ERTS_DIST_EXT_CON_ID_MASK; dep->prev = NULL; + erts_atomic64_set_nob(&dep->in, 0); + erts_atomic64_set_nob(&dep->out, 0); + erts_atomic32_set_nob(&dep->qflgs, + (is_internal_port(cid) + ? ERTS_DE_QFLG_PORT_CTRL + : ERTS_DE_QFLG_PROC_CTRL)); if(flags & DFLAG_PUBLISHED) { dep->next = erts_visible_dist_entries; if(erts_visible_dist_entries) { @@ -480,7 +628,7 @@ erts_set_dist_entry_connected(DistEntry *dep, Eterm cid, Uint flags) erts_hidden_dist_entries = dep; erts_no_of_hidden_dist_entries++; } - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_rwmtx_rwunlock(&erts_dist_table_rwmtx); } /* -- Node table --------------------------------------------------------- */ @@ -518,7 +666,7 @@ node_table_alloc(void *venp_tmpl) node_entries++; - erts_smp_refc_init(&enp->refc, -1); + erts_refc_init(&enp->refc, -1); enp->creation = ((ErlNode *) venp_tmpl)->creation; enp->sysname = ((ErlNode *) venp_tmpl)->sysname; enp->dist_entry = erts_find_or_insert_dist_entry(((ErlNode *) venp_tmpl)->sysname); @@ -531,7 +679,7 @@ node_table_free(void *venp) { ErlNode *enp = (ErlNode *) venp; - ERTS_SMP_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(enp != erts_this_node || erts_thr_progress_is_blocking()); erts_deref_dist_entry(enp->dist_entry); #ifdef DEBUG @@ -552,14 +700,14 @@ erts_node_table_size(void) #endif int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + erts_rwmtx_rlock(&erts_node_table_rwmtx); #ifdef DEBUG hash_get_info(&hi, &erts_node_table); ASSERT(node_entries == hi.objs); #endif res = hash_table_sz(&erts_node_table) + node_entries*sizeof(ErlNode); if (lock) - erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + erts_rwmtx_runlock(&erts_node_table_rwmtx); return res; } @@ -568,10 +716,10 @@ erts_node_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) - erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + erts_rwmtx_rlock(&erts_node_table_rwmtx); hash_info(to, to_arg, &erts_node_table); if (lock) - erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + erts_rwmtx_runlock(&erts_node_table_rwmtx); } @@ -582,26 +730,26 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint32 creation) ne.sysname = sysname; ne.creation = creation; - erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + erts_rwmtx_rlock(&erts_node_table_rwmtx); res = hash_get(&erts_node_table, (void *) &ne); if (res && res != erts_this_node) { - erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ - erts_smp_refc_inc(&res->refc, 1); + erts_refc_inc(&res->refc, 1); } - erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + erts_rwmtx_runlock(&erts_node_table_rwmtx); if (res) return res; - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - erts_aint_t refc = erts_smp_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ - erts_smp_refc_inc(&res->refc, 1); + erts_refc_inc(&res->refc, 1); } - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_rwmtx_rwunlock(&erts_node_table_rwmtx); return res; } @@ -610,7 +758,7 @@ static void try_delete_node(void *venp) ErlNode *enp = (ErlNode *) venp; erts_aint_t refc; - erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); + erts_rwmtx_rwlock(&erts_node_table_rwmtx); /* * Another thread might have looked up this node after we * decided to delete it (refc became zero). If so, the other @@ -626,10 +774,10 @@ static void try_delete_node(void *venp) * * If refc > 0, the entry is in use. Keep the entry. */ - refc = erts_smp_refc_dectest(&enp->refc, -1); + refc = erts_refc_dectest(&enp->refc, -1); if (refc == -1) (void) hash_erase(&erts_node_table, (void *) enp); - erts_smp_rwmtx_rwunlock(&erts_node_table_rwmtx); + erts_rwmtx_rwunlock(&erts_node_table_rwmtx); if (refc == 0) erts_schedule_delete_node(enp); @@ -672,7 +820,7 @@ static void print_node(void *venp, void *vpndp) erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", - erts_smp_refc_read(&enp->refc, 0)); + erts_refc_read(&enp->refc, 0)); #endif pndp->no_sysname++; } @@ -695,13 +843,13 @@ void erts_print_node_info(fmtfn_t to, pnd.no_total = 0; if (lock) - erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + erts_rwmtx_rlock(&erts_node_table_rwmtx); hash_foreach(&erts_node_table, print_node, (void *) &pnd); if (pnd.no_sysname != 0) { erts_print(to, to_arg, "\n"); } if (lock) - erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + erts_rwmtx_runlock(&erts_node_table_rwmtx); if(no_sysname) *no_sysname = pnd.no_sysname; @@ -714,20 +862,19 @@ void erts_print_node_info(fmtfn_t to, void erts_set_this_node(Eterm sysname, Uint creation) { - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); - ASSERT(erts_smp_refc_read(&erts_this_dist_entry->refc, 2)); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); + ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2)); - if (erts_smp_refc_dectest(&erts_this_node->refc, 0) == 0) + if (erts_refc_dectest(&erts_this_node->refc, 0) == 0) try_delete_node(erts_this_node); - if (erts_smp_refc_dectest(&erts_this_dist_entry->refc, 0) == 0) - try_delete_dist_entry(erts_this_dist_entry); + erts_deref_dist_entry(erts_this_dist_entry); erts_this_node = NULL; /* to make sure refc is bumped for this node */ erts_this_node = erts_find_or_insert_node(sysname, creation); erts_this_dist_entry = erts_this_node->dist_entry; - erts_smp_refc_inc(&erts_this_dist_entry->refc, 2); + erts_ref_dist_entry(erts_this_dist_entry); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), @@ -746,7 +893,7 @@ erts_delayed_node_table_gc(void) void erts_init_node_tables(int dd_sec) { - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; ErlNode node_tmpl; @@ -757,11 +904,13 @@ void erts_init_node_tables(int dd_sec) orig_node_tab_delete_delay = node_tab_delete_delay; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); - erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); + erts_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + erts_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; @@ -789,14 +938,14 @@ void erts_init_node_tables(int dd_sec) node_tmpl.creation = 0; erts_this_node = hash_put(&erts_node_table, &node_tmpl); /* +1 for erts_this_node */ - erts_smp_refc_init(&erts_this_node->refc, 1); + erts_refc_init(&erts_this_node->refc, 1); ASSERT(erts_this_node->dist_entry != NULL); erts_this_dist_entry = erts_this_node->dist_entry; /* +1 for erts_this_dist_entry */ - /* +1 for erts_this_node->dist_entry */ - erts_smp_refc_init(&erts_this_dist_entry->refc, 2); + erts_ref_dist_entry(erts_this_dist_entry); + ASSERT(2 == de_refc_read(erts_this_dist_entry, 2)); erts_this_node_sysname = erts_this_node_sysname_BUFFER; erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), @@ -805,17 +954,42 @@ void erts_init_node_tables(int dd_sec) references_atoms_need_init = 1; } -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK int erts_lc_is_de_rwlocked(DistEntry *dep) { - return erts_smp_lc_rwmtx_is_rwlocked(&dep->rwmtx); + return erts_lc_rwmtx_is_rwlocked(&dep->rwmtx); } int erts_lc_is_de_rlocked(DistEntry *dep) { - return erts_smp_lc_rwmtx_is_rlocked(&dep->rwmtx); + return erts_lc_rwmtx_is_rlocked(&dep->rwmtx); } #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT + +static void erts_lcnt_enable_dist_lock_count(void *dep_raw, void *enable) { + DistEntry *dep = (DistEntry*)dep_raw; + + if(enable) { + erts_lcnt_install_new_lock_info(&dep->rwmtx.lcnt, "dist_entry", dep->sysname, + ERTS_LOCK_TYPE_RWMUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + erts_lcnt_install_new_lock_info(&dep->lnk_mtx.lcnt, "dist_entry_links", dep->sysname, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + erts_lcnt_install_new_lock_info(&dep->qlock.lcnt, "dist_entry_out_queue", dep->sysname, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION); + } else { + erts_lcnt_uninstall(&dep->rwmtx.lcnt); + erts_lcnt_uninstall(&dep->lnk_mtx.lcnt); + erts_lcnt_uninstall(&dep->qlock.lcnt); + } +} + +void erts_lcnt_update_distribution_locks(int enable) { + erts_rwmtx_rlock(&erts_dist_table_rwmtx); + hash_foreach(&erts_dist_table, erts_lcnt_enable_dist_lock_count, + (void*)(UWord)enable); + erts_rwmtx_runlock(&erts_dist_table_rwmtx); +} #endif /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -848,6 +1022,7 @@ static Eterm AM_node_references; static Eterm AM_system; static Eterm AM_timer; static Eterm AM_delayed_delete_timer; +static Eterm AM_thread_progress_delete_timer; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); @@ -916,8 +1091,8 @@ erts_get_node_and_dist_references(struct process *proc) Uint *endp; #endif - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_block(); /* No need to lock any thing since we are alone... */ if (references_atoms_need_init) { @@ -937,6 +1112,7 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(timer); INIT_AM(system); INIT_AM(delayed_delete_timer); + INIT_AM(thread_progress_delete_timer); references_atoms_need_init = 0; } @@ -959,8 +1135,8 @@ erts_get_node_and_dist_references(struct process *proc) delete_reference_table(); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); return res; } @@ -1120,6 +1296,10 @@ insert_offheap2(ErlOffHeap *oh, void *arg) insert_offheap(oh, a->type, a->id); } +#define ErtsIsDistEntryBinary(Bin) \ + (((Bin)->intern.flags & BIN_FLAG_MAGIC) \ + && ERTS_MAGIC_BIN_DESTRUCTOR((Bin)) == erts_dist_entry_destructor) + static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { @@ -1130,7 +1310,10 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { switch (thing_subtag(u.hdr->thing_word)) { case REF_SUBTAG: - if(IsMatchProgBinary(u.mref->mb)) { + if (ErtsIsDistEntryBinary(u.mref->mb)) + insert_dist_entry(ErtsBin2DistEntry(u.mref->mb), + type, id, 0); + else if(IsMatchProgBinary(u.mref->mb)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) @@ -1262,39 +1445,45 @@ init_referred_dist(void *dist, void *unused) no_referred_dists++; } -#ifdef ERTS_SMP static void insert_sys_msg(Eterm from, Eterm to, Eterm msg, ErlHeapFragment *bp) { insert_offheap(&bp->off_heap, HEAP_REF, to); } -#endif static void insert_delayed_delete_node(void *state, ErtsMonotonicTime timeout_pos, void *vnp) { - DeclareTmpHeapNoproc(heap,3); - UseTmpHeapNoproc(3); + Eterm heap[3]; insert_node((ErlNode *) vnp, SYSTEM_REF, TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer)); - UnUseTmpHeapNoproc(3); +} + +static void +insert_thr_prgr_delete_dist_entry(void *arg, ErtsThrPrgrVal thr_prgr, void *vbin) +{ + DistEntry *dep = ErtsBin2DistEntry(vbin); + Eterm heap[3]; + insert_dist_entry(dep, + SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_thread_progress_delete_timer), + 0); } static void insert_delayed_delete_dist_entry(void *state, ErtsMonotonicTime timeout_pos, - void *vdep) + void *vbin) { - DeclareTmpHeapNoproc(heap,3); - UseTmpHeapNoproc(3); - insert_dist_entry((DistEntry *) vdep, + DistEntry *dep = ErtsBin2DistEntry(vbin); + Eterm heap[3]; + insert_dist_entry(dep, SYSTEM_REF, TUPLE2(&heap[0], AM_system, AM_delayed_delete_timer), 0); - UnUseTmpHeapNoproc(3); } static void @@ -1328,9 +1517,12 @@ setup_reference_table(void) erts_debug_callback_timer_foreach(try_delete_node, insert_delayed_delete_node, NULL); - erts_debug_callback_timer_foreach(try_delete_dist_entry, + erts_debug_callback_timer_foreach(prepare_try_delete_dist_entry, insert_delayed_delete_dist_entry, NULL); + erts_debug_later_op_foreach(try_delete_dist_entry, + insert_thr_prgr_delete_dist_entry, + NULL); UseTmpHeapNoproc(3); insert_node(erts_this_node, @@ -1351,9 +1543,7 @@ setup_reference_table(void) int mli; ErtsMessage *msg_list[] = { proc->msg.first, -#ifdef ERTS_SMP proc->msg_inq.first, -#endif proc->msg_frag}; /* Insert Heap */ @@ -1397,12 +1587,18 @@ setup_reference_table(void) insert_links(ERTS_P_LINKS(proc), proc->common.id); if (ERTS_P_MONITORS(proc)) insert_monitors(ERTS_P_MONITORS(proc), proc->common.id); + { + DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); + if (dep) + insert_dist_entry(dep, + CTRL_REF, + proc->common.id, + 0); + } } } -#ifdef ERTS_SMP erts_foreach_sys_msg_in_q(insert_sys_msg); -#endif /* Insert all ports */ max = erts_ptab_max(&erts_port); @@ -1636,7 +1832,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) tup = MK_2TUP(referred_nodes[i].node->sysname, MK_UINT(referred_nodes[i].node->creation)); - tup = MK_3TUP(tup, MK_UINT(erts_smp_refc_read(&referred_nodes[i].node->refc, 0)), nril); + tup = MK_3TUP(tup, MK_UINT(erts_refc_read(&referred_nodes[i].node->refc, 0)), nril); nl = MK_CONS(tup, nl); } @@ -1697,7 +1893,7 @@ reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp) /* DistList = [{Dist, Refc, ReferenceIdList}] */ tup = MK_3TUP(referred_dists[i].dist->sysname, - MK_UINT(erts_smp_refc_read(&referred_dists[i].dist->refc, 0)), + MK_UINT(de_refc_read(referred_dists[i].dist, 0)), dril); dl = MK_CONS(tup, dl); } @@ -1756,12 +1952,12 @@ delete_reference_table(void) void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs) { - erts_smp_thr_progress_block(); + erts_thr_progress_block(); if (millisecs < 0) node_tab_delete_delay = orig_node_tab_delete_delay; else node_tab_delete_delay = millisecs; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 489da1ba17..3bba673435 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -44,10 +44,12 @@ #include "erl_alloc.h" #include "erl_process.h" #include "erl_monitors.h" -#include "erl_smp.h" #define ERTS_PORT_TASK_ONLY_BASIC_TYPES__ #include "erl_port_task.h" #undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ +#define ERTS_BINARY_TYPES_ONLY__ +#include "erl_binary.h" +#undef ERTS_BINARY_TYPES_ONLY__ #define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60) #define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000) @@ -61,11 +63,17 @@ #define ERTS_DE_SFLGS_ALL (ERTS_DE_SFLG_CONNECTED \ | ERTS_DE_SFLG_EXITING) -#define ERTS_DE_QFLG_BUSY (((Uint32) 1) << 0) -#define ERTS_DE_QFLG_EXIT (((Uint32) 1) << 1) +#define ERTS_DE_QFLG_BUSY (((erts_aint32_t) 1) << 0) +#define ERTS_DE_QFLG_EXIT (((erts_aint32_t) 1) << 1) +#define ERTS_DE_QFLG_REQ_INFO (((erts_aint32_t) 1) << 2) +#define ERTS_DE_QFLG_PORT_CTRL (((erts_aint32_t) 1) << 3) +#define ERTS_DE_QFLG_PROC_CTRL (((erts_aint32_t) 1) << 4) #define ERTS_DE_QFLGS_ALL (ERTS_DE_QFLG_BUSY \ - | ERTS_DE_QFLG_EXIT) + | ERTS_DE_QFLG_EXIT \ + | ERTS_DE_QFLG_REQ_INFO \ + | ERTS_DE_QFLG_PORT_CTRL \ + | ERTS_DE_QFLG_PROC_CTRL) #if defined(ARCH_64) #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL) @@ -107,12 +115,13 @@ typedef struct dist_entry_ { HashBucket hash_bucket; /* Hash bucket */ struct dist_entry_ *next; /* Next entry in dist_table (not sorted) */ struct dist_entry_ *prev; /* Previous entry in dist_table (not sorted) */ - erts_smp_refc_t refc; /* Reference count */ - erts_smp_rwmtx_t rwmtx; /* Protects all fields below until lck_mtx. */ + erts_rwmtx_t rwmtx; /* Protects all fields below until lck_mtx. */ Eterm sysname; /* name@host atom for efficiency */ Uint32 creation; /* creation of connected node */ - Eterm cid; /* connection handler (pid or port), NIL == free */ + erts_atomic_t input_handler; /* Input handler */ + Eterm cid; /* connection handler (pid or port), + NIL == free */ Uint32 connection_id; /* Connection id incremented on connect */ Uint32 status; /* Slot status, like exiting reserved etc */ Uint32 flags; /* Distribution flags, like hidden, @@ -120,7 +129,7 @@ typedef struct dist_entry_ { unsigned long version; /* Protocol version */ - erts_smp_mtx_t lnk_mtx; /* Protects node_links, nlinks, and + erts_mtx_t lnk_mtx; /* Protects node_links, nlinks, and monitors. */ ErtsLink *node_links; /* In a dist entry, node links are kept in a separate tree, while they are @@ -132,24 +141,29 @@ typedef struct dist_entry_ { ErtsLink *nlinks; /* Link tree with subtrees */ ErtsMonitor *monitors; /* Monitor tree */ - erts_smp_mtx_t qlock; /* Protects qflgs and out_queue */ - Uint32 qflgs; - Sint qsize; + erts_mtx_t qlock; /* Protects qflgs and out_queue */ + erts_atomic32_t qflgs; + erts_atomic_t qsize; + erts_atomic64_t in; + erts_atomic64_t out; ErtsDistOutputQueue out_queue; struct ErtsProcList_ *suspended; + ErtsDistOutputQueue tmp_out_queue; ErtsDistOutputQueue finalized_out_queue; - erts_smp_atomic_t dist_cmd_scheduled; + erts_atomic_t dist_cmd_scheduled; ErtsPortTaskHandle dist_cmd; Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf); struct cache* cache; /* The atom cache */ + + ErtsThrPrgrLaterOp later_op; } DistEntry; typedef struct erl_node_ { HashBucket hash_bucket; /* Hash bucket */ - erts_smp_refc_t refc; /* Reference count */ + erts_refc_t refc; /* Reference count */ Eterm sysname; /* name@host atom for efficiency */ Uint32 creation; /* Creation */ DistEntry *dist_entry; /* Corresponding dist entry */ @@ -158,8 +172,8 @@ typedef struct erl_node_ { extern Hash erts_dist_table; extern Hash erts_node_table; -extern erts_smp_rwmtx_t erts_dist_table_rwmtx; -extern erts_smp_rwmtx_t erts_node_table_rwmtx; +extern erts_rwmtx_t erts_dist_table_rwmtx; +extern erts_rwmtx_t erts_node_table_rwmtx; extern DistEntry *erts_hidden_dist_entries; extern DistEntry *erts_visible_dist_entries; @@ -190,72 +204,68 @@ void erts_init_node_tables(int); void erts_node_table_info(fmtfn_t, void *); void erts_print_node_info(fmtfn_t, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int erts_lc_is_de_rwlocked(DistEntry *); int erts_lc_is_de_rlocked(DistEntry *); #endif +int erts_dist_entry_destructor(Binary *bin); +DistEntry *erts_dhandle_to_dist_entry(Eterm dhandle); +Eterm erts_make_dhandle(Process *c_p, DistEntry *dep); +void erts_ref_dist_entry(DistEntry *dep); +void erts_deref_dist_entry(DistEntry *dep); -ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np); -ERTS_GLB_INLINE void erts_smp_de_rlock(DistEntry *dep); -ERTS_GLB_INLINE void erts_smp_de_runlock(DistEntry *dep); -ERTS_GLB_INLINE void erts_smp_de_rwlock(DistEntry *dep); -ERTS_GLB_INLINE void erts_smp_de_rwunlock(DistEntry *dep); -ERTS_GLB_INLINE void erts_smp_de_links_lock(DistEntry *dep); -ERTS_GLB_INLINE void erts_smp_de_links_unlock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_links_lock(DistEntry *dep); +ERTS_GLB_INLINE void erts_de_links_unlock(DistEntry *dep); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_deref_dist_entry(DistEntry *dep) -{ - ASSERT(dep); - if (erts_smp_refc_dectest(&dep->refc, 0) == 0) - erts_schedule_delete_dist_entry(dep); -} - -ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np) { ASSERT(np); - if (erts_smp_refc_dectest(&np->refc, 0) == 0) + if (erts_refc_dectest(&np->refc, 0) == 0) erts_schedule_delete_node(np); } ERTS_GLB_INLINE void -erts_smp_de_rlock(DistEntry *dep) +erts_de_rlock(DistEntry *dep) { - erts_smp_rwmtx_rlock(&dep->rwmtx); + erts_rwmtx_rlock(&dep->rwmtx); } ERTS_GLB_INLINE void -erts_smp_de_runlock(DistEntry *dep) +erts_de_runlock(DistEntry *dep) { - erts_smp_rwmtx_runlock(&dep->rwmtx); + erts_rwmtx_runlock(&dep->rwmtx); } ERTS_GLB_INLINE void -erts_smp_de_rwlock(DistEntry *dep) +erts_de_rwlock(DistEntry *dep) { - erts_smp_rwmtx_rwlock(&dep->rwmtx); + erts_rwmtx_rwlock(&dep->rwmtx); } ERTS_GLB_INLINE void -erts_smp_de_rwunlock(DistEntry *dep) +erts_de_rwunlock(DistEntry *dep) { - erts_smp_rwmtx_rwunlock(&dep->rwmtx); + erts_rwmtx_rwunlock(&dep->rwmtx); } ERTS_GLB_INLINE void -erts_smp_de_links_lock(DistEntry *dep) +erts_de_links_lock(DistEntry *dep) { - erts_smp_mtx_lock(&dep->lnk_mtx); + erts_mtx_lock(&dep->lnk_mtx); } ERTS_GLB_INLINE void -erts_smp_de_links_unlock(DistEntry *dep) +erts_de_links_unlock(DistEntry *dep) { - erts_smp_mtx_unlock(&dep->lnk_mtx); + erts_mtx_unlock(&dep->lnk_mtx); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 6a3213ec52..9117eb1f72 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -31,6 +31,9 @@ typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData; #include "erl_ptab.h" #include "erl_thr_progress.h" #include "erl_trace.h" +#define ERTS_IO_QUEUE_TYPES_ONLY__ +#include "erl_io_queue.h" +#undef ERTS_IO_QUEUE_TYPES_ONLY__ #ifndef __WIN32__ #define ERTS_DEFAULT_MAX_PORTS (1 << 16) @@ -75,23 +78,8 @@ typedef struct erts_driver_t_ erts_driver_t; #define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH)) #endif -#define SMALL_IO_QUEUE 5 /* Number of fixed elements */ +typedef ErtsIOQueue ErlPortIOQueue; -typedef struct { - ErlDrvSizeT size; /* total size in bytes */ - - SysIOVec* v_start; - SysIOVec* v_end; - SysIOVec* v_head; - SysIOVec* v_tail; - SysIOVec v_small[SMALL_IO_QUEUE]; - - ErlDrvBinary** b_start; - ErlDrvBinary** b_end; - ErlDrvBinary** b_head; - ErlDrvBinary** b_tail; - ErlDrvBinary* b_small[SMALL_IO_QUEUE]; -} ErlIOQueue; typedef struct line_buf { /* Buffer used in line oriented I/O */ ErlDrvSizeT bufsiz; /* Size of character buffer */ @@ -131,9 +119,7 @@ typedef struct { void *data[ERTS_PRTSD_SIZE]; } ErtsPrtSD; -#ifdef ERTS_SMP typedef struct ErtsXPortsList_ ErtsXPortsList; -#endif /* * Port locking: @@ -158,21 +144,16 @@ struct _erl_drv_port { ErtsPortTaskSched sched; ErtsPortTaskHandle timeout_task; -#ifdef ERTS_SMP erts_mtx_t *lock; ErtsXPortsList *xports; - erts_smp_atomic_t run_queue; -#else - erts_atomic32_t refc; - int cleanup; -#endif + erts_atomic_t run_queue; erts_atomic_t connected; /* A connected process */ Eterm caller; /* Current caller. */ - erts_smp_atomic_t data; /* Data associated with port. */ + erts_atomic_t data; /* Data associated with port. */ Uint bytes_in; /* Number of bytes read */ Uint bytes_out; /* Number of bytes written */ - ErlIOQueue ioq; /* driver accessible i/o queue */ + ErlPortIOQueue ioq; /* driver accessible i/o queue */ DistEntry *dist_entry; /* Dist entry used in DISTRIBUTION */ char *name; /* String used in the open */ erts_driver_t* drv_ptr; @@ -185,7 +166,7 @@ struct _erl_drv_port { int control_flags; /* Flags for port_control() */ ErlDrvPDL port_data_lock; - erts_smp_atomic_t psd; /* Port specific data */ + erts_atomic_t psd; /* Port specific data */ int reds; /* Only used while executing driver callbacks */ struct { @@ -221,24 +202,20 @@ ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt); ERTS_GLB_INLINE ErtsRunQueue * erts_port_runq(Port *prt) { -#ifdef ERTS_SMP ErtsRunQueue *rq1, *rq2; - rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); + rq1 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); if (!rq1) return NULL; while (1) { - erts_smp_runq_lock(rq1); - rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue); + erts_runq_lock(rq1); + rq2 = (ErtsRunQueue *) erts_atomic_read_nob(&prt->run_queue); if (rq1 == rq2) return rq1; - erts_smp_runq_unlock(rq1); + erts_runq_unlock(rq1); rq1 = rq2; if (!rq1) return NULL; } -#else - return ERTS_RUNQ_IX(0); -#endif } #endif @@ -252,10 +229,10 @@ ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new); ERTS_GLB_INLINE void * erts_prtsd_get(Port *prt, int ix) { - ErtsPrtSD *psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + ErtsPrtSD *psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); if (!psd) return NULL; - ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; return psd->data[ix]; } @@ -266,16 +243,14 @@ erts_prtsd_set(Port *prt, int ix, void *data) void *old; int i; - psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); if (psd) { -#ifdef ERTS_SMP #ifdef ETHR_ORDERED_READ_DEPEND ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); #else ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); #endif -#endif old = psd->data[ix]; psd->data[ix] = data; return old; @@ -287,7 +262,7 @@ erts_prtsd_set(Port *prt, int ix, void *data) new_psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD)); for (i = 0; i < ERTS_PRTSD_SIZE; i++) new_psd->data[i] = NULL; - psd = (ErtsPrtSD *) erts_smp_atomic_cmpxchg_mb(&prt->psd, + psd = (ErtsPrtSD *) erts_atomic_cmpxchg_mb(&prt->psd, (erts_aint_t) new_psd, (erts_aint_t) NULL); if (psd) @@ -371,15 +346,10 @@ Eterm erts_request_io_bytes(Process *c_p); void print_port_info(Port *, fmtfn_t, void *); void erts_port_free(Port *); -#ifndef ERTS_SMP -void erts_port_cleanup(Port *); -#endif void erts_fire_port_monitor(Port *prt, Eterm ref); -#ifdef ERTS_SMP int erts_port_handle_xports(Port *); -#endif -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int erts_lc_is_port_locked(Port *); #endif @@ -388,9 +358,9 @@ ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt); ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc); ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt); -ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt); -ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt); -ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt); +ERTS_GLB_INLINE int erts_port_trylock(Port *prt); +ERTS_GLB_INLINE void erts_port_lock(Port *prt); +ERTS_GLB_INLINE void erts_port_unlock(Port *prt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -419,35 +389,27 @@ ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt) } ERTS_GLB_INLINE int -erts_smp_port_trylock(Port *prt) +erts_port_trylock(Port *prt) { -#ifdef ERTS_SMP /* *Need* to be a managed thread */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); return erts_mtx_trylock(prt->lock); -#else - return 0; -#endif } ERTS_GLB_INLINE void -erts_smp_port_lock(Port *prt) +erts_port_lock(Port *prt) { -#ifdef ERTS_SMP /* *Need* to be a managed thread */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); erts_mtx_lock(prt->lock); -#endif } ERTS_GLB_INLINE void -erts_smp_port_unlock(Port *prt) +erts_port_unlock(Port *prt) { -#ifdef ERTS_SMP /* *Need* to be a managed thread */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); erts_mtx_unlock(prt->lock); -#endif } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ @@ -478,9 +440,7 @@ extern const Port erts_invalid_port; int erts_is_port_ioq_empty(Port *); void erts_terminate_port(Port *); -#ifdef ERTS_SMP Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks); -#endif ERTS_GLB_INLINE Port *erts_pix2port(int); ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm); @@ -488,11 +448,9 @@ ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32); ERTS_GLB_INLINE Port*erts_id2port(Eterm id); ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32); ERTS_GLB_INLINE void erts_port_release(Port *); -#ifdef ERTS_SMP ERTS_GLB_INLINE Port *erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs); ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs); ERTS_GLB_INLINE void erts_thr_port_release(Port *prt); -#endif ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int); ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *); ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort); @@ -518,7 +476,7 @@ erts_port_lookup_raw(Eterm id) { Port *prt; - ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying()); if (is_not_internal_port(id)) return NULL; @@ -547,7 +505,7 @@ erts_id2port(Eterm id) Port *prt; /* Only allowed to be called from managed threads */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); if (is_not_internal_port(id)) return NULL; @@ -558,10 +516,10 @@ erts_id2port(Eterm id) if (!prt || prt->common.id != id) return NULL; - erts_smp_port_lock(prt); + erts_port_lock(prt); state = erts_atomic32_read_nob(&prt->state); if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) { - erts_smp_port_unlock(prt); + erts_port_unlock(prt); return NULL; } @@ -574,14 +532,12 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 invalid_sflgs) { -#ifdef ERTS_SMP int no_proc_locks = !c_p || !c_p_locks; -#endif erts_aint32_t state; Port *prt; /* Only allowed to be called from managed threads */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); if (is_not_internal_port(id)) return NULL; @@ -592,21 +548,17 @@ erts_id2port_sflgs(Eterm id, if (!prt || prt->common.id != id) return NULL; -#ifdef ERTS_SMP if (no_proc_locks) - erts_smp_port_lock(prt); - else if (erts_smp_port_trylock(prt) == EBUSY) { + erts_port_lock(prt); + else if (erts_port_trylock(prt) == EBUSY) { /* Unlock process locks, and acquire locks in lock order... */ - erts_smp_proc_unlock(c_p, c_p_locks); - erts_smp_port_lock(prt); - erts_smp_proc_lock(c_p, c_p_locks); + erts_proc_unlock(c_p, c_p_locks); + erts_port_lock(prt); + erts_proc_lock(c_p, c_p_locks); } -#endif state = erts_atomic32_read_nob(&prt->state); if (state & invalid_sflgs) { -#ifdef ERTS_SMP - erts_smp_port_unlock(prt); -#endif + erts_port_unlock(prt); return NULL; } @@ -617,18 +569,10 @@ ERTS_GLB_INLINE void erts_port_release(Port *prt) { /* Only allowed to be called from managed threads */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); -#ifdef ERTS_SMP - erts_smp_port_unlock(prt); -#else - if (prt->cleanup) { - prt->cleanup = 0; - erts_port_cleanup(prt); - } -#endif + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); + erts_port_unlock(prt); } -#ifdef ERTS_SMP /* * erts_thr_id2port_sflgs() and erts_port_dec_refc(prt) can * be used by unmanaged threads in the SMP case. @@ -714,13 +658,10 @@ ERTS_GLB_INLINE void erts_thr_port_release(Port *prt) { erts_mtx_unlock(prt->lock); -#ifdef ERTS_SMP if (!erts_thr_progress_is_managed_thread()) erts_port_dec_refc(prt); -#endif } -#endif ERTS_GLB_INLINE Port * erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) @@ -736,7 +677,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) #ifdef ERTS_ENABLE_LOCK_CHECK if (!ERTS_IS_CRASH_DUMPING) { if (erts_lc_is_emu_thr()) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ERTS_LC_ASSERT(!prt->port_data_lock || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx)); } @@ -765,7 +706,7 @@ erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep) // ERTS_LC_ASSERT(erts_lc_is_emu_thr()); if (prt == ERTS_INVALID_ERL_DRV_PORT) return ERTS_INVALID_ERL_DRV_PORT; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); /* * This state check is only needed since a driver callback @@ -822,23 +763,21 @@ erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep) int reds = 0; erts_aint32_t state; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); state = erts_atomic32_read_nob(&prt->state); if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) { reds += ERTS_PORT_REDS_TERMINATE; erts_terminate_port(prt); state = erts_atomic32_read_nob(&prt->state); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); } -#ifdef ERTS_SMP if (prt->xports) { reds += erts_port_handle_xports(prt); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(!prt->xports); } -#endif if (statep) *statep = state; diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index a044de3fee..a588477320 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -36,6 +36,7 @@ #include "erl_check_io.h" #include "dtrace-wrapper.h" #include "lttng-wrapper.h" +#include "erl_check_io.h" #include <stdarg.h> /* @@ -83,15 +84,13 @@ static void chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_q #define LTTNG_DRIVER(TRACEPOINT, PP) do {} while(0) #endif -#define ERTS_SMP_LC_VERIFY_RQ(RQ, PP) \ +#define ERTS_LC_VERIFY_RQ(RQ, PP) \ do { \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); \ - ERTS_SMP_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ - erts_smp_atomic_read_nob(&(PP)->run_queue))); \ + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); \ + ERTS_LC_ASSERT((RQ) == ((ErtsRunQueue *) \ + erts_atomic_read_nob(&(PP)->run_queue))); \ } while (0) -erts_smp_atomic_t erts_port_task_outstanding_io_tasks; - #define ERTS_PT_STATE_SCHEDULED 0 #define ERTS_PT_STATE_ABORTED 1 #define ERTS_PT_STATE_EXECUTING 2 @@ -99,7 +98,6 @@ erts_smp_atomic_t erts_port_task_outstanding_io_tasks; typedef union { struct { /* I/O tasks */ ErlDrvEvent event; - ErlDrvEventData event_data; } io; struct { ErtsProc2PortSigCallback callback; @@ -108,7 +106,7 @@ typedef union { } ErtsPortTaskTypeData; struct ErtsPortTask_ { - erts_smp_atomic32_t state; + erts_atomic32_t state; ErtsPortTaskType type; union { struct { @@ -126,9 +124,7 @@ struct ErtsPortTaskHandleList_ { ErtsPortTaskHandle handle; union { ErtsPortTaskHandleList *next; -#ifdef ERTS_SMP ErtsThrPrgrLaterOp release; -#endif } u; }; @@ -151,35 +147,29 @@ static void begin_port_cleanup(Port *pp, ErtsPortTask **execq, int *processing_busy_q_p); -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_task, - ErtsPortTask, - 1000, - ERTS_ALC_T_PORT_TASK) +ERTS_THR_PREF_QUICK_ALLOC_IMPL(port_task, + ErtsPortTask, + 1000, + ERTS_ALC_T_PORT_TASK) ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(busy_caller_table, ErtsPortTaskBusyCallerTable, 50, ERTS_ALC_T_BUSY_CALLER_TAB) -#ifdef ERTS_SMP static void call_port_task_free(void *vptp) { port_task_free((ErtsPortTask *) vptp); } -#endif static ERTS_INLINE void schedule_port_task_free(ErtsPortTask *ptp) { -#ifdef ERTS_SMP erts_schedule_thr_prgr_later_cleanup_op(call_port_task_free, (void *) ptp, &ptp->u.release, sizeof(ErtsPortTask)); -#else - port_task_free(ptp); -#endif } static ERTS_INLINE ErtsPortTask * @@ -199,7 +189,7 @@ p2p_sig_data_init(ErtsPortTask *ptp) ptp->type = ERTS_PORT_TASK_PROC_SIG; ptp->u.alive.flags = ERTS_PT_FLG_SIG_DEP; - erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); + erts_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); ASSERT(ptp == p2p_sig_data_to_task(&ptp->u.alive.td.psig.data)); @@ -290,7 +280,7 @@ popped_from_busy_queue(Port *pp, ErtsPortTask *ptp, int last) #ifdef DEBUG erts_aint32_t flags = #endif - erts_smp_atomic32_read_band_nob( + erts_atomic32_read_band_nob( &pp->sched.flags, ~ERTS_PTS_FLG_HAVE_BUSY_TASKS); ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS); @@ -337,7 +327,7 @@ busy_wait_move_to_busy_queue(Port *pp, ErtsPortTask *ptp) #ifdef DEBUG flags = #endif - erts_smp_atomic32_read_bor_nob(&pp->sched.flags, + erts_atomic32_read_bor_nob(&pp->sched.flags, ERTS_PTS_FLG_HAVE_BUSY_TASKS); ASSERT(!(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS)); @@ -477,7 +467,7 @@ no_sig_dep_move_from_busyq(Port *pp) int bix; erts_aint32_t flags = #endif - erts_smp_atomic32_read_band_nob( + erts_atomic32_read_band_nob( &pp->sched.flags, ~ERTS_PTS_FLG_HAVE_BUSY_TASKS); ASSERT(flags & ERTS_PTS_FLG_HAVE_BUSY_TASKS); @@ -510,11 +500,11 @@ chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue) if (!first) { ASSERT(!tabp); ASSERT(!pp->sched.taskq.local.busy.last); - ASSERT(!(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS)); + ASSERT(!(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS)); return; } - ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS); + ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_HAVE_BUSY_TASKS); ASSERT(tabp); tot_count = 0; @@ -570,13 +560,13 @@ chk_task_queues(Port *pp, ErtsPortTask *execq, int processing_busy_queue) static ERTS_INLINE void reset_port_task_handle(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_set_relb(pthp, (erts_aint_t) NULL); + erts_atomic_set_relb(pthp, (erts_aint_t) NULL); } static ERTS_INLINE ErtsPortTask * handle2task(ErtsPortTaskHandle *pthp) { - return (ErtsPortTask *) erts_smp_atomic_read_acqb(pthp); + return (ErtsPortTask *) erts_atomic_read_acqb(pthp); } static ERTS_INLINE void @@ -593,8 +583,9 @@ reset_executed_io_task_handle(ErtsPortTask *ptp) { if (ptp->u.alive.handle) { ASSERT(ptp == handle2task(ptp->u.alive.handle)); - erts_io_notify_port_task_executed(ptp->u.alive.handle); - reset_port_task_handle(ptp->u.alive.handle); + /* The port task handle is reset inside task_executed */ + erts_io_notify_port_task_executed(ptp->type, ptp->u.alive.handle, + reset_port_task_handle); } } @@ -603,7 +594,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->u.alive.handle = pthp; if (pthp) { - erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + erts_atomic_set_relb(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->u.alive.handle)); } } @@ -617,7 +608,7 @@ set_tmp_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) * IMPORTANT! Task either need to be aborted, or task handle * need to be detached before thread progress has been made. */ - erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + erts_atomic_set_relb(pthp, (erts_aint_t) ptp); } } @@ -635,20 +626,20 @@ check_unset_busy_port_q(Port *pp, int resume_procs = 0; ASSERT(bpq); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); erts_port_task_sched_lock(&pp->sched); - qsize = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size); - low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low); + qsize = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->size); + low = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low); if (qsize < low) { erts_aint32_t mask = ~(ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q | ERTS_PTS_FLG_BUSY_PORT_Q); - flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags, mask); + flags = erts_atomic32_read_band_relb(&pp->sched.flags, mask); if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q) resume_procs = 1; } else if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) { - flags = erts_smp_atomic32_read_band_relb(&pp->sched.flags, + flags = erts_atomic32_read_band_relb(&pp->sched.flags, ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); flags &= ~ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q; } @@ -673,16 +664,16 @@ aborted_proc2port_data(Port *pp, ErlDrvSizeT size) bpq = pp->sched.taskq.bpq; - qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size, + qsz = (ErlDrvSizeT) erts_atomic_add_read_acqb(&bpq->size, (erts_aint_t) -size); ASSERT(qsz + size > qsz); - flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + flags = erts_atomic32_read_nob(&pp->sched.flags); ASSERT(pp->sched.taskq.bpq); if ((flags & (ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q | ERTS_PTS_FLG_BUSY_PORT_Q)) != ERTS_PTS_FLG_BUSY_PORT_Q) return; - if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low)) - erts_smp_atomic32_read_bor_nob(&pp->sched.flags, + if (qsz < (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low)) + erts_atomic32_read_bor_nob(&pp->sched.flags, ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); } @@ -700,13 +691,13 @@ dequeued_proc2port_data(Port *pp, ErlDrvSizeT size) bpq = pp->sched.taskq.bpq; - qsz = (ErlDrvSizeT) erts_smp_atomic_add_read_acqb(&bpq->size, + qsz = (ErlDrvSizeT) erts_atomic_add_read_acqb(&bpq->size, (erts_aint_t) -size); ASSERT(qsz + size > qsz); - flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + flags = erts_atomic32_read_nob(&pp->sched.flags); if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q)) return; - if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->low)) + if (qsz < (ErlDrvSizeT) erts_atomic_read_acqb(&bpq->low)) check_unset_busy_port_q(pp, flags, bpq); } @@ -719,19 +710,19 @@ enqueue_proc2port_data(Port *pp, if (sigdp && bpq) { ErlDrvSizeT size = erts_proc2port_sig_command_data_size(sigdp); if (size) { - erts_aint_t asize = erts_smp_atomic_add_read_acqb(&bpq->size, + erts_aint_t asize = erts_atomic_add_read_acqb(&bpq->size, (erts_aint_t) size); ErlDrvSizeT qsz = (ErlDrvSizeT) asize; ASSERT(qsz - size < qsz); if (!(flags & ERTS_PTS_FLG_BUSY_PORT_Q) && qsz > bpq->high) { - flags = erts_smp_atomic32_read_bor_acqb(&pp->sched.flags, + flags = erts_atomic32_read_bor_acqb(&pp->sched.flags, ERTS_PTS_FLG_BUSY_PORT_Q); flags |= ERTS_PTS_FLG_BUSY_PORT_Q; - qsz = (ErlDrvSizeT) erts_smp_atomic_read_acqb(&bpq->size); - if (qsz < (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low)) { - flags = (erts_smp_atomic32_read_bor_relb( + qsz = (ErlDrvSizeT) erts_atomic_read_acqb(&bpq->size); + if (qsz < (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low)) { + flags = (erts_atomic32_read_bor_relb( &pp->sched.flags, ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q)); flags |= ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q; @@ -779,18 +770,18 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp erts_aint32_t flags; pp->sched.taskq.bpq = NULL; flags = ~(ERTS_PTS_FLG_BUSY_PORT_Q|ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); - flags = erts_smp_atomic32_read_band_acqb(&pp->sched.flags, flags); + flags = erts_atomic32_read_band_acqb(&pp->sched.flags, flags); if ((flags & ERTS_PTS_FLGS_BUSY) == ERTS_PTS_FLG_BUSY_PORT_Q) resume_procs = 1; } else { if (!low) - low = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->low); + low = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->low); else { if (bpq->high < low) bpq->high = low; - erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low); + erts_atomic_set_relb(&bpq->low, (erts_aint_t) low); written = 1; } @@ -799,19 +790,19 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp else { if (low > high) { low = high; - erts_smp_atomic_set_relb(&bpq->low, (erts_aint_t) low); + erts_atomic_set_relb(&bpq->low, (erts_aint_t) low); } bpq->high = high; written = 1; } if (written) { - ErlDrvSizeT size = (ErlDrvSizeT) erts_smp_atomic_read_nob(&bpq->size); + ErlDrvSizeT size = (ErlDrvSizeT) erts_atomic_read_nob(&bpq->size); if (size > high) - erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_BUSY_PORT_Q); else if (size < low) - erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q); } } @@ -830,32 +821,27 @@ erl_drv_busy_msgq_limits(ErlDrvPort dport, ErlDrvSizeT *lowp, ErlDrvSizeT *highp * No-suspend handles. */ -#ifdef ERTS_SMP static void free_port_task_handle_list(void *vpthlp) { erts_free(ERTS_ALC_T_PT_HNDL_LIST, vpthlp); } -#endif static void schedule_port_task_handle_list_free(ErtsPortTaskHandleList *pthlp) { -#ifdef ERTS_SMP erts_schedule_thr_prgr_later_cleanup_op(free_port_task_handle_list, (void *) pthlp, &pthlp->u.release, sizeof(ErtsPortTaskHandleList)); -#else - erts_free(ERTS_ALC_T_PT_HNDL_LIST, pthlp); -#endif } static ERTS_INLINE void -abort_nosuspend_task(Port *pp, - ErtsPortTaskType type, - ErtsPortTaskTypeData *tdp, - int bpq_data) +abort_signal_task(Port *pp, + int abort_type, + ErtsPortTaskType type, + ErtsPortTaskTypeData *tdp, + int bpq_data) { ASSERT(type == ERTS_PORT_TASK_PROC_SIG); @@ -863,24 +849,34 @@ abort_nosuspend_task(Port *pp, if (!bpq_data) tdp->psig.callback(NULL, ERTS_PORT_SFLG_INVALID, - ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, + abort_type, &tdp->psig.data); else { ErlDrvSizeT size = erts_proc2port_sig_command_data_size(&tdp->psig.data); tdp->psig.callback(NULL, ERTS_PORT_SFLG_INVALID, - ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, + abort_type, &tdp->psig.data); aborted_proc2port_data(pp, size); } } + +static ERTS_INLINE void +abort_nosuspend_task(Port *pp, + ErtsPortTaskType type, + ErtsPortTaskTypeData *tdp, + int bpq_data) +{ + abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND, type, tdp, bpq_data); +} + static ErtsPortTaskHandleList * get_free_nosuspend_handles(Port *pp) { ErtsPortTaskHandleList *nshp, *last_nshp = NULL; - ERTS_SMP_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched)); + ERTS_LC_ASSERT(erts_port_task_sched_lock_is_locked(&pp->sched)); nshp = pp->sched.taskq.local.busy.nosuspend; @@ -896,7 +892,7 @@ get_free_nosuspend_handles(Port *pp) pp->sched.taskq.local.busy.nosuspend = last_nshp->u.next; last_nshp->u.next = NULL; if (!pp->sched.taskq.local.busy.nosuspend) - erts_smp_atomic32_read_band_nob(&pp->sched.flags, + erts_atomic32_read_band_nob(&pp->sched.flags, ~ERTS_PTS_FLG_HAVE_NS_TASKS); } return nshp; @@ -919,7 +915,7 @@ free_nosuspend_handles(ErtsPortTaskHandleList *free_nshp) static ERTS_INLINE void enqueue_port(ErtsRunQueue *runq, Port *pp) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); pp->sched.next = NULL; if (runq->ports.end) { ASSERT(runq->ports.start); @@ -933,19 +929,17 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) runq->ports.end = pp; ASSERT(runq->ports.start && runq->ports.end); - erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); + erts_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); -#ifdef ERTS_SMP if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING) erts_non_empty_runq(runq); -#endif } static ERTS_INLINE Port * pop_port(ErtsRunQueue *runq) { Port *pp = runq->ports.start; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); if (!pp) { ASSERT(!runq->ports.end); } @@ -955,7 +949,7 @@ pop_port(ErtsRunQueue *runq) ASSERT(runq->ports.end == pp); runq->ports.end = NULL; } - erts_smp_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); + erts_dec_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); } ASSERT(runq->ports.start || !runq->ports.end); @@ -982,7 +976,7 @@ enqueue_task(Port *pp, if (ns_pthlp) fail_flags |= ERTS_PTS_FLG_BUSY_PORT; erts_port_task_sched_lock(&pp->sched); - flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + flags = erts_atomic32_read_nob(&pp->sched.flags); if (flags & fail_flags) res = 0; else { @@ -1013,7 +1007,7 @@ enqueue_task(Port *pp, static ERTS_INLINE void prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) { - erts_aint32_t act = erts_smp_atomic32_read_nob(&pp->sched.flags); + erts_aint32_t act = erts_atomic32_read_nob(&pp->sched.flags); if (!pp->sched.taskq.local.busy.first || (act & ERTS_PTS_FLG_BUSY_PORT)) { *execqp = pp->sched.taskq.local.first; @@ -1034,7 +1028,7 @@ prepare_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) new &= ~ERTS_PTS_FLG_IN_RUNQ; new |= ERTS_PTS_FLG_EXEC; - act = erts_smp_atomic32_cmpxchg_nob(&pp->sched.flags, new, exp); + act = erts_atomic32_cmpxchg_nob(&pp->sched.flags, new, exp); ASSERT(act & ERTS_PTS_FLG_IN_RUNQ); @@ -1061,7 +1055,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) *execq = NULL; - act = erts_smp_atomic32_read_nob(&pp->sched.flags); + act = erts_atomic32_read_nob(&pp->sched.flags); if (act & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) act = check_unset_busy_port_q(pp, act, pp->sched.taskq.bpq); @@ -1078,7 +1072,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) if (act & ERTS_PTS_FLG_HAVE_TASKS) new |= ERTS_PTS_FLG_IN_RUNQ; - act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); + act = erts_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_IN_RUNQ)); ERTS_LC_ASSERT(!(act & ERTS_PTS_FLG_EXEC_IMM)); @@ -1104,7 +1098,7 @@ finalize_exec(Port *pp, ErtsPortTask **execq, int processing_busy_q) static ERTS_INLINE erts_aint32_t select_queue_for_exec(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) { - erts_aint32_t flags = erts_smp_atomic32_read_nob(&pp->sched.flags); + erts_aint32_t flags = erts_atomic32_read_nob(&pp->sched.flags); if (flags & ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q) flags = check_unset_busy_port_q(pp, flags, pp->sched.taskq.bpq); @@ -1214,7 +1208,7 @@ fetch_in_queue(Port *pp, ErtsPortTask **execqp) if (ptp) *execqp = ptp->u.alive.next; else - erts_smp_atomic32_read_band_nob(&pp->sched.flags, + erts_atomic32_read_band_nob(&pp->sched.flags, ~ERTS_PTS_FLG_HAVE_TASKS); @@ -1277,7 +1271,7 @@ erl_drv_consume_timeslice(ErlDrvPort dprt, int percent) void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *pthp) { - ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying()); reset_port_task_handle(pthp); } @@ -1290,9 +1284,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) { int res; ErtsPortTask *ptp; -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); -#endif ptp = handle2task(pthp); if (!ptp) @@ -1302,41 +1294,25 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) #ifdef DEBUG ErtsPortTaskHandle *saved_pthp = ptp->u.alive.handle; - ERTS_SMP_READ_MEMORY_BARRIER; - old_state = erts_smp_atomic32_read_nob(&ptp->state); + ERTS_THR_READ_MEMORY_BARRIER; + old_state = erts_atomic32_read_nob(&ptp->state); if (old_state == ERTS_PT_STATE_SCHEDULED) { ASSERT(!saved_pthp || saved_pthp == pthp); } #endif - old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + old_state = erts_atomic32_cmpxchg_nob(&ptp->state, ERTS_PT_STATE_ABORTED, ERTS_PT_STATE_SCHEDULED); if (old_state != ERTS_PT_STATE_SCHEDULED) res = - 1; /* Task already aborted, executing, or executed */ else { - reset_port_task_handle(pthp); - - switch (ptp->type) { - case ERTS_PORT_TASK_INPUT: - case ERTS_PORT_TASK_OUTPUT: - case ERTS_PORT_TASK_EVENT: - ASSERT(erts_smp_atomic_read_nob( - &erts_port_task_outstanding_io_tasks) > 0); - erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); - break; - default: - break; - } - res = 0; } } -#ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); -#endif return res; } @@ -1345,12 +1321,10 @@ void erts_port_task_abort_nosuspend_tasks(Port *pp) { ErtsPortTaskHandleList *abort_list; -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl = ERTS_THR_PRGR_DHANDLE_INVALID; -#endif erts_port_task_sched_lock(&pp->sched); - erts_smp_atomic32_read_band_nob(&pp->sched.flags, + erts_atomic32_read_band_nob(&pp->sched.flags, ~ERTS_PTS_FLG_HAVE_NS_TASKS); abort_list = pp->sched.taskq.local.busy.nosuspend; pp->sched.taskq.local.busy.nosuspend = NULL; @@ -1370,40 +1344,34 @@ erts_port_task_abort_nosuspend_tasks(Port *pp) pthlp = abort_list; abort_list = pthlp->u.next; -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) dhndl = erts_thr_progress_unmanaged_delay(); -#endif pthp = &pthlp->handle; ptp = handle2task(pthp); if (!ptp) { -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_thr_progress_unmanaged_continue(dhndl); -#endif schedule_port_task_handle_list_free(pthlp); continue; } #ifdef DEBUG saved_pthp = ptp->u.alive.handle; - ERTS_SMP_READ_MEMORY_BARRIER; - old_state = erts_smp_atomic32_read_nob(&ptp->state); + ERTS_THR_READ_MEMORY_BARRIER; + old_state = erts_atomic32_read_nob(&ptp->state); if (old_state == ERTS_PT_STATE_SCHEDULED) { ASSERT(saved_pthp == pthp); } #endif - old_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + old_state = erts_atomic32_cmpxchg_nob(&ptp->state, ERTS_PT_STATE_ABORTED, ERTS_PT_STATE_SCHEDULED); if (old_state != ERTS_PT_STATE_SCHEDULED) { /* Task already aborted, executing, or executed */ -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_thr_progress_unmanaged_continue(dhndl); -#endif schedule_port_task_handle_list_free(pthlp); continue; } @@ -1413,10 +1381,8 @@ erts_port_task_abort_nosuspend_tasks(Port *pp) type = ptp->type; td = ptp->u.alive.td; -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_thr_progress_unmanaged_continue(dhndl); -#endif schedule_port_task_handle_list_free(pthlp); abort_nosuspend_task(pp, type, &td, pp->sched.taskq.bpq != NULL); @@ -1435,10 +1401,8 @@ erts_port_task_schedule(Eterm id, { ErtsProc2PortSigData *sigdp = NULL; ErtsPortTaskHandleList *ns_pthlp = NULL; -#ifdef ERTS_SMP ErtsRunQueue *xrunq; ErtsThrPrgrDelayHandle dhndl; -#endif ErtsRunQueue *runq; Port *pp; ErtsPortTask *ptp = NULL; @@ -1449,30 +1413,26 @@ erts_port_task_schedule(Eterm id, ASSERT(is_internal_port(id)); -#ifdef ERTS_SMP dhndl = erts_thr_progress_unmanaged_delay(); -#endif pp = erts_port_lookup_raw(id); -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { if (pp) erts_port_inc_refc(pp); erts_thr_progress_unmanaged_continue(dhndl); } -#endif - - if (!pp) - goto fail; if (type != ERTS_PORT_TASK_PROC_SIG) { + if (!pp) + goto fail; + ptp = port_task_alloc(); ptp->type = type; ptp->u.alive.flags = 0; - erts_smp_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); + erts_atomic32_init_nob(&ptp->state, ERTS_PT_STATE_SCHEDULED); set_handle(ptp, pthp); } @@ -1484,16 +1444,6 @@ erts_port_task_schedule(Eterm id, va_start(argp, type); ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); va_end(argp); - erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); - break; - } - case ERTS_PORT_TASK_EVENT: { - va_list argp; - va_start(argp, type); - ptp->u.alive.td.io.event = va_arg(argp, ErlDrvEvent); - ptp->u.alive.td.io.event_data = va_arg(argp, ErlDrvEventData); - va_end(argp); - erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks); break; } case ERTS_PORT_TASK_PROC_SIG: { @@ -1504,6 +1454,9 @@ erts_port_task_schedule(Eterm id, ptp->u.alive.td.psig.callback = va_arg(argp, ErtsProc2PortSigCallback); ptp->u.alive.flags |= va_arg(argp, int); va_end(argp); + if (!pp) + goto fail; + if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) set_tmp_handle(ptp, pthp); else { @@ -1545,7 +1498,7 @@ erts_port_task_schedule(Eterm id, if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) new |= ERTS_PTS_FLG_IN_RUNQ; - act = erts_smp_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); + act = erts_atomic32_cmpxchg_relb(&pp->sched.flags, new, exp); if (exp == act) { if (!(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) @@ -1570,67 +1523,63 @@ erts_port_task_schedule(Eterm id, if (!runq) ERTS_INTERNAL_ERROR("Missing run-queue"); -#ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); - ERTS_SMP_LC_ASSERT(runq != xrunq); - ERTS_SMP_LC_VERIFY_RQ(runq, pp); + ERTS_LC_ASSERT(runq != xrunq); + ERTS_LC_VERIFY_RQ(runq, pp); if (xrunq) { /* Emigrate port ... */ - erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); - erts_smp_runq_unlock(runq); + erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_runq_unlock(runq); runq = erts_port_runq(pp); if (!runq) ERTS_INTERNAL_ERROR("Missing run-queue"); } -#endif enqueue_port(runq, pp); - erts_smp_runq_unlock(runq); + erts_runq_unlock(runq); - erts_smp_notify_inc_runq(runq); + erts_notify_inc_runq(runq); done: if (prof_runnable_ports) erts_port_task_sched_unlock(&pp->sched); -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_port_dec_refc(pp); -#endif return 0; abort_nosuspend: -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_port_dec_refc(pp); -#endif abort_nosuspend_task(pp, ptp->type, &ptp->u.alive.td, 0); ASSERT(ns_pthlp); erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); - if (ptp) - port_task_free(ptp); + + ASSERT(ptp); + port_task_free(ptp); return 0; fail: -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) erts_port_dec_refc(pp); -#endif + + if (ptp) { + abort_signal_task(pp, ERTS_PROC2PORT_SIG_ABORT, + ptp->type, &ptp->u.alive.td, 0); + port_task_free(ptp); + } if (ns_pthlp) erts_free(ERTS_ALC_T_PT_HNDL_LIST, ns_pthlp); - if (ptp) - port_task_free(ptp); - return -1; } @@ -1640,14 +1589,14 @@ erts_port_task_free_port(Port *pp) erts_aint32_t flags; ErtsRunQueue *runq; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); runq = erts_port_runq(pp); if (!runq) ERTS_INTERNAL_ERROR("Missing run-queue"); erts_port_task_sched_lock(&pp->sched); - flags = erts_smp_atomic32_read_bor_relb(&pp->sched.flags, + flags = erts_atomic32_read_bor_relb(&pp->sched.flags, ERTS_PTS_FLG_EXIT); erts_port_task_sched_unlock(&pp->sched); erts_atomic32_read_bset_relb(&pp->state, @@ -1657,7 +1606,7 @@ erts_port_task_free_port(Port *pp) | ERTS_PORT_SFLG_FREE), ERTS_PORT_SFLG_FREE); - erts_smp_runq_unlock(runq); + erts_runq_unlock(runq); if (!(flags & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) begin_port_cleanup(pp, NULL, NULL); @@ -1670,13 +1619,12 @@ erts_port_task_free_port(Port *pp) * scheduling of processes. Run-queue lock should be held by caller. */ -int +void erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { Port *pp; ErtsPortTask *execq; int processing_busy_q; - int res = 0; int vreds = 0; int reds = 0; erts_aint_t io_tasks_executed = 0; @@ -1687,17 +1635,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ErtsSchedulerData *esdp = runq->scheduler; ERTS_MSACC_PUSH_STATE_M(); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); pp = pop_port(runq); if (!pp) { - res = 0; goto done; } - ERTS_SMP_LC_VERIFY_RQ(runq, pp); + ERTS_LC_VERIFY_RQ(runq, pp); - erts_smp_runq_unlock(runq); + erts_runq_unlock(runq); *curr_port_pp = pp; @@ -1705,19 +1652,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no); int migrated = old && old != esdp->no; - erts_smp_spin_lock(&erts_sched_stat.lock); + erts_spin_lock(&erts_sched_stat.lock); erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_executed++; erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].executed++; if (migrated) { erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_migrated++; erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].migrated++; } - erts_smp_spin_unlock(&erts_sched_stat.lock); + erts_spin_unlock(&erts_sched_stat.lock); } prepare_exec(pp, &execq, &processing_busy_q); - erts_smp_port_lock(pp); + erts_port_lock(pp); /* trace port scheduling, in */ if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { @@ -1739,7 +1686,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) if (!ptp) break; - task_state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + task_state = erts_atomic32_cmpxchg_nob(&ptp->state, ERTS_PT_STATE_EXECUTING, ERTS_PT_STATE_SCHEDULED); if (task_state != ERTS_PT_STATE_SCHEDULED) { @@ -1751,8 +1698,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) start_time = erts_timestamp_millis(); } - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_CHK_NO_PROC_LOCKS; ASSERT(pp->drv_ptr); switch (ptp->type) { @@ -1794,17 +1741,6 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reset_executed_io_task_handle(ptp); io_tasks_executed++; break; - case ERTS_PORT_TASK_EVENT: - reds = ERTS_PORT_REDS_EVENT; - ASSERT((state & ERTS_PORT_SFLGS_DEAD) == 0); - DTRACE_DRIVER(driver_event, pp); - LTTNG_DRIVER(driver_event, pp); - (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, - ptp->u.alive.td.io.event, - ptp->u.alive.td.io.event_data); - reset_executed_io_task_handle(ptp); - io_tasks_executed++; - break; case ERTS_PORT_TASK_PROC_SIG: { ErtsProc2PortSigData *sigdp = &ptp->u.alive.td.psig.data; reset_handle(ptp); @@ -1869,17 +1805,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); - - if (io_tasks_executed) { - ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - >= io_tasks_executed); - erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks, - -1*io_tasks_executed); - } - -#ifdef ERTS_SMP - ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); -#endif + ASSERT(runq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); active = finalize_exec(pp, &execq, processing_busy_q); @@ -1889,54 +1815,43 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) *curr_port_pp = NULL; - erts_smp_runq_lock(runq); + erts_runq_lock(runq); if (active) { -#ifdef ERTS_SMP ErtsRunQueue *xrunq; -#endif ASSERT(!(erts_atomic32_read_nob(&pp->state) & ERTS_PORT_SFLGS_DEAD)); -#ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); - ERTS_SMP_LC_ASSERT(runq != xrunq); - ERTS_SMP_LC_VERIFY_RQ(runq, pp); + ERTS_LC_ASSERT(runq != xrunq); + ERTS_LC_VERIFY_RQ(runq, pp); if (!xrunq) { -#endif enqueue_port(runq, pp); /* No need to notify ourselves about inc in runq. */ -#ifdef ERTS_SMP } else { /* Emigrate port... */ - erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); - erts_smp_runq_unlock(runq); + erts_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); + erts_runq_unlock(runq); xrunq = erts_port_runq(pp); ASSERT(xrunq); enqueue_port(xrunq, pp); - erts_smp_runq_unlock(xrunq); - erts_smp_notify_inc_runq(xrunq); + erts_runq_unlock(xrunq); + erts_notify_inc_runq(xrunq); - erts_smp_runq_lock(runq); + erts_runq_lock(runq); } -#endif } done: - res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) - != (erts_aint_t) 0); runq->scheduler->reductions += reds; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); ERTS_PORT_REDUCTIONS_EXECUTED(esdp, runq, reds); - - return res; } -#ifdef ERTS_SMP static void release_port(void *vport) { @@ -1952,7 +1867,6 @@ schedule_release_port(void *vport) { &pp->common.u.release); } -#endif static void begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) @@ -1963,7 +1877,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) ErtsPortTaskHandleList *free_nshp = NULL; ErtsProcList *plp; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); /* * Abort remaining tasks... @@ -2036,11 +1950,11 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) qs[i] = ptp->u.alive.next; /* Normal case here is aborted tasks... */ - state = erts_smp_atomic32_read_nob(&ptp->state); + state = erts_atomic32_read_nob(&ptp->state); if (state == ERTS_PT_STATE_ABORTED) goto aborted_port_task; - state = erts_smp_atomic32_cmpxchg_nob(&ptp->state, + state = erts_atomic32_cmpxchg_nob(&ptp->state, ERTS_PT_STATE_EXECUTING, ERTS_PT_STATE_SCHEDULED); if (state != ERTS_PT_STATE_SCHEDULED) { @@ -2067,13 +1981,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) DO_WRITE, 1); break; - case ERTS_PORT_TASK_EVENT: - erts_stale_drv_select(pp->common.id, - ERTS_Port2ErlDrvPort(pp), - ptp->u.alive.td.io.event, - 0, - 1); - break; case ERTS_PORT_TASK_DIST_CMD: break; case ERTS_PORT_TASK_PROC_SIG: { @@ -2104,7 +2011,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) } } - erts_smp_atomic32_read_band_nob(&pp->sched.flags, + erts_atomic32_read_band_nob(&pp->sched.flags, ~(ERTS_PTS_FLG_HAVE_BUSY_TASKS |ERTS_PTS_FLG_HAVE_TASKS |ERTS_PTS_FLGS_BUSY)); @@ -2146,7 +2053,6 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) /* * Schedule cleanup of port structure... */ -#ifdef ERTS_SMP /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */ if (!erts_get_scheduler_data()) { erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp); @@ -2156,19 +2062,15 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) (void *) pp, &pp->common.u.release); } -#else - pp->cleanup = 1; -#endif } -#ifdef ERTS_SMP void erts_enqueue_port(ErtsRunQueue *rq, Port *pp) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - ASSERT(rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); - ASSERT(erts_smp_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); + ASSERT(rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ); enqueue_port(rq, pp); } @@ -2176,16 +2078,15 @@ Port * erts_dequeue_port(ErtsRunQueue *rq) { Port *pp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); pp = pop_port(rq); ASSERT(!pp - || rq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); - ASSERT(!pp || (erts_smp_atomic32_read_nob(&pp->sched.flags) + || rq == (ErtsRunQueue *) erts_atomic_read_nob(&pp->run_queue)); + ASSERT(!pp || (erts_atomic32_read_nob(&pp->sched.flags) & ERTS_PTS_FLG_IN_RUNQ)); return pp; } -#endif /* * Initialize the module. @@ -2193,8 +2094,7 @@ erts_dequeue_port(ErtsRunQueue *rq) void erts_port_task_init(void) { - erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks, - (erts_aint_t) 0); - init_port_task_alloc(); + init_port_task_alloc(erts_no_schedulers + erts_no_poll_threads + + 1); /* aux_thread */ init_busy_caller_table_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 9cca62ffaf..ae78a7d8a3 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -27,11 +27,11 @@ #ifndef ERTS_PORT_TASK_H_BASIC_TYPES__ #define ERTS_PORT_TASK_H_BASIC_TYPES__ #include "erl_sys_driver.h" -#include "erl_smp.h" +#include "erl_threads.h" #define ERL_PORT_GET_PORT_TYPE_ONLY__ #include "erl_port.h" #undef ERL_PORT_GET_PORT_TYPE_ONLY__ -typedef erts_smp_atomic_t ErtsPortTaskHandle; +typedef erts_atomic_t ErtsPortTaskHandle; #endif #ifndef ERTS_PORT_TASK_ONLY_BASIC_TYPES__ @@ -56,17 +56,11 @@ typedef erts_smp_atomic_t ErtsPortTaskHandle; typedef enum { ERTS_PORT_TASK_INPUT, ERTS_PORT_TASK_OUTPUT, - ERTS_PORT_TASK_EVENT, ERTS_PORT_TASK_TIMEOUT, ERTS_PORT_TASK_DIST_CMD, ERTS_PORT_TASK_PROC_SIG } ErtsPortTaskType; -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -/* NOTE: Do not access any of the exported variables directly */ -extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; -#endif - #define ERTS_PTS_FLG_IN_RUNQ (((erts_aint32_t) 1) << 0) #define ERTS_PTS_FLG_EXEC (((erts_aint32_t) 1) << 1) #define ERTS_PTS_FLG_HAVE_TASKS (((erts_aint32_t) 1) << 2) @@ -98,8 +92,8 @@ extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks; typedef struct { ErlDrvSizeT high; - erts_smp_atomic_t low; - erts_smp_atomic_t size; + erts_atomic_t low; + erts_atomic_t size; } ErtsPortTaskBusyPortQ; typedef struct ErtsPortTask_ ErtsPortTask; @@ -124,10 +118,8 @@ typedef struct { } in; ErtsPortTaskBusyPortQ *bpq; } taskq; - erts_smp_atomic32_t flags; -#ifdef ERTS_SMP + erts_atomic32_t flags; erts_mtx_t mtx; -#endif } ErtsPortTaskSched; ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp); @@ -142,22 +134,18 @@ ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp); ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); -#endif - #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL); + erts_atomic_init_nob(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp) { - return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL; + return ((void *) erts_atomic_read_acqb(pthp)) != NULL; } ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, @@ -165,9 +153,9 @@ ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, { if (bpq) { erts_aint_t low = (erts_aint_t) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW; - erts_smp_atomic_init_nob(&bpq->low, low); + erts_atomic_init_nob(&bpq->low, low); bpq->high = (ErlDrvSizeT) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH; - erts_smp_atomic_init_nob(&bpq->size, (erts_aint_t) 0); + erts_atomic_init_nob(&bpq->size, (erts_aint_t) 0); } ptsp->taskq.bpq = bpq; } @@ -175,9 +163,7 @@ ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp, ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) { -#ifdef ERTS_SMP char *lock_str = "port_sched_lock"; -#endif ptsp->next = NULL; ptsp->taskq.local.busy.first = NULL; ptsp->taskq.local.busy.last = NULL; @@ -186,38 +172,26 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) ptsp->taskq.local.first = NULL; ptsp->taskq.in.first = NULL; ptsp->taskq.in.last = NULL; - erts_smp_atomic32_init_nob(&ptsp->flags, 0); -#ifdef ERTS_SMP -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_mtx_init_x_opt(&ptsp->mtx, lock_str, instr_id, - ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) - ? 0 : ERTS_LCNT_LT_DISABLE)); -#else - erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); -#endif -#endif + erts_atomic32_init_nob(&ptsp->flags, 0); + erts_mtx_init(&ptsp->mtx, lock_str, instr_id, ERTS_LOCK_FLAGS_CATEGORY_IO); } ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp) { -#ifdef ERTS_SMP erts_mtx_lock(&ptsp->mtx); -#endif } ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp) { -#ifdef ERTS_SMP erts_mtx_unlock(&ptsp->mtx); -#endif } ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp) { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) return erts_lc_mtx_is_locked(&ptsp->mtx); #else return 0; @@ -228,35 +202,25 @@ erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp) { -#ifdef ERTS_SMP erts_mtx_destroy(&ptsp->mtx); -#endif } ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp) { - erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING); + erts_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING); } -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS - -ERTS_GLB_INLINE int -erts_port_task_have_outstanding_io_tasks(void) -{ - return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks) - != 0); -} - -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -int erts_port_task_execute(ErtsRunQueue *, Port **); +void erts_port_task_execute(ErtsRunQueue *, Port **); void erts_port_task_init(void); #endif +/* generated for 'port_task' quick allocator */ +void erts_port_task_pre_alloc_init_thread(void); + void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *); int erts_port_task_abort(ErtsPortTaskHandle *); @@ -271,10 +235,8 @@ ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void); ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data_extra(size_t extra, void **extra_ptr); void erts_port_task_free_p2p_sig_data(ErtsProc2PortSigData *sigdp); -#ifdef ERTS_SMP void erts_enqueue_port(ErtsRunQueue *rq, Port *pp); Port *erts_dequeue_port(ErtsRunQueue *rq); -#endif #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #endif /* ERL_PORT_TASK_H__ */ #endif /* ERTS_PORT_TASK_ONLY_BASIC_TYPES__ */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index fc2b34e70f..61fdf86a56 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -24,6 +24,8 @@ # include "config.h" #endif +#define ERTS_WANT_BREAK_HANDLING + #include <stddef.h> /* offsetof() */ #include "sys.h" #include "erl_vm.h" @@ -49,6 +51,8 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_nfunc_sched.h" +#include "erl_check_io.h" +#include "erl_poll.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -127,18 +131,16 @@ runq_got_work_to_execute_flags(Uint32 flags) return !ERTS_IS_RUNQ_EMPTY_FLGS(flags); } -#ifdef ERTS_SMP static ERTS_INLINE int runq_got_work_to_execute(ErtsRunQueue *rq) { return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); } -#endif #undef RUNQ_READ_RQ #undef RUNQ_SET_RQ -#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_smp_atomic_read_nob((X))) -#define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) +#define RUNQ_READ_RQ(X) ((ErtsRunQueue *) erts_atomic_read_nob((X))) +#define RUNQ_SET_RQ(X, RQ) erts_atomic_set_nob((X), (erts_aint_t) (RQ)) #ifdef DEBUG # if defined(ARCH_64) @@ -172,7 +174,6 @@ extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = SPO_ON_HEAP_MSGQ; -int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); @@ -194,57 +195,51 @@ static UWord thr_prgr_later_cleanup_op_threshold = ERTS_THR_PRGR_LATER_CLEANUP_O ErtsPTab erts_proc erts_align_attribute(ERTS_CACHE_LINE_SIZE); int erts_sched_thread_suggested_stack_size = -1; -#ifdef ERTS_DIRTY_SCHEDULERS int erts_dcpu_sched_thread_suggested_stack_size = -1; int erts_dio_sched_thread_suggested_stack_size = -1; -#endif #ifdef ERTS_ENABLE_LOCK_CHECK ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif -static struct { +static struct ErtsSchedBusyWait_ { int aux_work; int tse; int sys_schedule; } sched_busy_wait; -#ifdef ERTS_SMP int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; +static ErtsAuxWorkData *poll_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -typedef struct { +typedef struct ErtsMultiSchedulingBlock_ { int ongoing; ErtsProcList *blckrs; ErtsProcList *chngq; } ErtsMultiSchedulingBlock; -typedef struct { +typedef struct ErtsSchedTypeCounters_ { Uint32 normal; -#ifdef ERTS_DIRTY_SCHEDULERS Uint32 dirty_cpu; Uint32 dirty_io; -#endif } ErtsSchedTypeCounters; -static struct { - erts_smp_mtx_t mtx; +static struct ErtsSchedSuspend_ { + erts_mtx_t mtx; ErtsSchedTypeCounters online; ErtsSchedTypeCounters curr_online; ErtsSchedTypeCounters active; - erts_smp_atomic32_t changing; + erts_atomic32_t changing; ErtsProcList *chngq; Eterm changer; ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ -#ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedType last_msb_dirty_type; -#endif } schdlr_sspnd; static void init_scheduler_suspend(void); @@ -253,10 +248,8 @@ static ERTS_INLINE Uint32 schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p) { int res = val1p->normal == val2p->normal; -#ifdef ERTS_DIRTY_SCHEDULERS res &= val1p->dirty_cpu == val2p->dirty_cpu; res &= val1p->dirty_io == val2p->dirty_io; -#endif return res; } @@ -267,16 +260,10 @@ schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, switch (type) { case ERTS_SCHED_NORMAL: return valp->normal; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: return valp->dirty_cpu; case ERTS_SCHED_DIRTY_IO: return valp->dirty_io; -#else - case ERTS_SCHED_DIRTY_CPU: - case ERTS_SCHED_DIRTY_IO: - return 0; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return 0; @@ -288,10 +275,8 @@ static ERTS_INLINE Uint32 schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) { Uint32 res = valp->normal; -#ifdef ERTS_DIRTY_SCHEDULERS res += valp->dirty_cpu; res += valp->dirty_io; -#endif return res; } #endif @@ -306,14 +291,12 @@ schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal--; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu--; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io--; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } @@ -327,14 +310,12 @@ schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal++; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu++; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io++; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } @@ -348,25 +329,23 @@ schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp, case ERTS_SCHED_NORMAL: valp->normal = no; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: valp->dirty_cpu = no; break; case ERTS_SCHED_DIRTY_IO: valp->dirty_io = no; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static struct { - erts_smp_mtx_t update_mtx; - erts_smp_atomic32_t no_runqs; + erts_mtx_t update_mtx; + erts_atomic32_t no_runqs; int last_active_runqs; int forced_check_balance; - erts_smp_atomic32_t checking_balance; + erts_atomic32_t checking_balance; int halftime; int full_reds_history_index; struct { @@ -384,51 +363,38 @@ do { \ balance_info.prev_rise.reds = (REDS); \ } while (0) -#endif erts_sched_stat_t erts_sched_stat; -#ifdef USE_THREADS static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key); -#endif -static erts_smp_atomic32_t function_calls; - -#ifdef ERTS_SMP -static erts_smp_atomic32_t doing_sys_schedule; -static erts_smp_atomic32_t no_empty_run_queues; +static erts_atomic32_t no_empty_run_queues; long erts_runq_supervision_interval = 0; static ethr_event runq_supervision_event; static erts_tid_t runq_supervisor_tid; static erts_atomic_t runq_supervisor_sleeping; -#else /* !ERTS_SMP */ -ErtsSchedulerData *erts_scheduler_data; -#endif ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues); Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues); -#ifdef ERTS_DIRTY_SCHEDULERS struct { union { - erts_smp_atomic32_t active; + erts_atomic32_t active; char align__[ERTS_CACHE_LINE_SIZE]; } cpu; union { - erts_smp_atomic32_t active; + erts_atomic32_t active; char align__[ERTS_CACHE_LINE_SIZE]; } io; } dirty_count erts_align_attribute(ERTS_CACHE_LINE_SIZE); -#endif static ERTS_INLINE void dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) { -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t val; - erts_smp_atomic32_t *ap; + erts_atomic32_t *ap; switch (esdp->type) { case ERTS_SCHED_DIRTY_CPU: ap = &dirty_count.cpu.active; @@ -446,23 +412,20 @@ dirty_active(ErtsSchedulerData *esdp, erts_aint32_t add) * All updates done under run-queue lock, so * no inc or dec needed... */ - ERTS_SMP_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(esdp->run_queue)); - val = erts_smp_atomic32_read_nob(ap); + val = erts_atomic32_read_nob(ap); val += add; - erts_smp_atomic32_set_nob(ap, val); -#endif + erts_atomic32_set_nob(ap, val); } ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data); -#ifdef ERTS_DIRTY_SCHEDULERS ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data); ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data); typedef union { Process dsp; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))]; } ErtsAlignedDirtyShadowProcess; -#endif typedef union { ErtsSchedulerSleepInfo ssi; @@ -470,12 +433,9 @@ typedef union { } ErtsAlignedSchedulerSleepInfo; static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP static ErtsAlignedSchedulerSleepInfo *aligned_dirty_cpu_sched_sleep_info; static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; -#endif -#endif +static ErtsAlignedSchedulerSleepInfo *aligned_poll_thread_sleep_info; static Uint last_reductions; static Uint last_exact_reductions; @@ -543,11 +503,14 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) +#define ERTS_POLL_THREAD_SLEEP_INFO_IX(IX) \ + (ASSERT(0 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_poll_threads)), \ + &aligned_poll_thread_sleep_info[(IX)].ssi) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT(-1 <= ((int) (IX)) \ - && ((int) (IX)) < ((int) erts_no_schedulers)), \ + (ASSERT(((int)-1) <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_schedulers)), \ &aligned_sched_sleep_info[(IX)].ssi) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX) \ (ASSERT(0 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_dirty_cpu_schedulers)), \ @@ -556,7 +519,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, (ASSERT(0 <= ((int) (IX)) \ && ((int) (IX)) < ((int) erts_no_dirty_io_schedulers)), \ &aligned_dirty_io_sched_sleep_info[(IX)].ssi) -#endif #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -564,9 +526,9 @@ do { \ int ix__; \ for (ix__ = 0; ix__ < erts_no_run_queues; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ - erts_smp_runq_lock(RQVAR); \ + erts_runq_lock(RQVAR); \ { DO; } \ - erts_smp_runq_unlock(RQVAR); \ + erts_runq_unlock(RQVAR); \ } \ } while (0) @@ -576,12 +538,12 @@ do { \ int ix__; \ int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, \ ERTS_SCHED_NORMAL); \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ for (ix__ = 0; ix__ < online__; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ - erts_smp_runq_lock(RQVAR); \ + erts_runq_lock(RQVAR); \ { DO; } \ - erts_smp_runq_unlock(RQVAR); \ + erts_runq_unlock(RQVAR); \ } \ } while (0) @@ -592,12 +554,12 @@ do { \ int ix__; \ for (ix__ = 0; ix__ < nrqs; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ - erts_smp_runq_lock(RQVAR); \ + erts_runq_lock(RQVAR); \ { DO; } \ } \ { DOX; } \ for (ix__ = 0; ix__ < nrqs; ix__++) \ - erts_smp_runq_unlock(ERTS_RUNQ_IX(ix__)); \ + erts_runq_unlock(ERTS_RUNQ_IX(ix__)); \ } while (0) #define ERTS_ATOMIC_FOREACH_RUNQ(RQVAR, DO) \ @@ -638,11 +600,8 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_MISC; valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; -#if ERTS_USE_ASYNC_READY_Q valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif -#ifdef ERTS_SMP valid |= ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP; valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; @@ -651,7 +610,6 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS; -#endif #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif @@ -673,16 +631,14 @@ dbg_chk_aux_work_val(erts_aint32_t value) #define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) #endif -#ifdef ERTS_SMP static void do_handle_pending_exiters(ErtsProcList *); static void wake_scheduler(ErtsRunQueue *rq); -#endif -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int -erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) +erts_lc_runq_is_locked(ErtsRunQueue *runq) { - return erts_smp_lc_mtx_is_locked(&runq->mtx); + return erts_lc_mtx_is_locked(&runq->mtx); } #endif @@ -690,13 +646,13 @@ erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) static ERTS_INLINE Uint64 ensure_later_proc_interval(Uint64 interval) { - return erts_smp_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval); + return erts_ensure_later_interval_nob(erts_ptab_interval(&erts_proc), interval); } Uint64 erts_get_proc_interval(void) { - return erts_smp_current_interval_nob(erts_ptab_interval(&erts_proc)); + return erts_current_interval_nob(erts_ptab_interval(&erts_proc)); } Uint64 @@ -708,15 +664,13 @@ erts_ensure_later_proc_interval(Uint64 interval) Uint64 erts_step_proc_interval(void) { - return erts_smp_step_interval_nob(erts_ptab_interval(&erts_proc)); + return erts_step_interval_nob(erts_ptab_interval(&erts_proc)); } void erts_pre_init_process(void) { -#ifdef USE_THREADS erts_tsd_key_create(&sched_data_key, "erts_sched_data_key"); -#endif erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX] = "DELAYED_AW_WAKEUP"; @@ -796,6 +750,11 @@ erts_pre_init_process(void) = ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_ETS_FIXED_TABLES].set_locks = ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks + = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks + = ERTS_PSD_DIST_ENTRY_SET_LOCKS; #endif } @@ -810,10 +769,8 @@ void erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) { -#ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; erts_init_proc_lock(ncpu); -#endif init_proclist_alloc(); @@ -825,11 +782,7 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) sizeof(Process), "process_table", legacy_proc_tab, -#ifdef ERTS_SMP 1 -#else - 0 -#endif ); last_reductions = 0; @@ -841,7 +794,9 @@ erts_late_init_process(void) { int ix; - erts_smp_spinlock_init(&erts_sched_stat.lock, "sched_stat"); + erts_spinlock_init(&erts_sched_stat.lock, "sched_stat", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); + for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { Eterm atom; char *atom_str; @@ -881,7 +836,6 @@ erts_late_init_process(void) static void init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) { -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp->type != ERTS_SCHED_NORMAL) { erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0); esdp->sched_wall_time.enabled = 1; @@ -890,7 +844,6 @@ init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; } else -#endif { esdp->sched_wall_time.u.need = erts_sched_balance_util; esdp->sched_wall_time.enabled = 0; @@ -1039,14 +992,14 @@ erts_get_sched_util(ErtsRunQueue *rq, int initially_locked, int short_interval) if (!locked) { if (++try >= ERTS_GET_AVG_MAX_UNLOCKED_TRY) { /* Writer will eventually block on runq-lock */ - erts_smp_runq_lock(rq); + erts_runq_lock(rq); locked = 1; } } } if (!initially_locked && locked) - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); now = sched_wall_time_ts(); worktime = calc_sched_worktime(is_working, now, last, interval, old_worktime); @@ -1088,7 +1041,6 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled) #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */ -#ifdef ERTS_DIRTY_SCHEDULERS typedef struct { Uint64 working; @@ -1141,9 +1093,7 @@ read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info info->working = info->total; } -#endif -#ifdef ERTS_SMP static void dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working) @@ -1191,16 +1141,13 @@ dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working) mod++; erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); -#if 0 if (!working) { - ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_BUSY_WAIT); } else { - ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + ERTS_MSACC_SET_STATE_X(ERTS_MSACC_STATE_OTHER); } -#endif } -#endif /* ERTS_SMP */ static void sched_wall_time_change(ErtsSchedulerData *esdp, int working) @@ -1245,11 +1192,9 @@ typedef struct { Eterm ref; Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; - erts_smp_atomic32_t refc; -#ifdef ERTS_DIRTY_SCHEDULERS + erts_atomic32_t refc; int want_dirty_cpu; int want_dirty_io; -#endif } ErtsSchedWallTimeReq; typedef struct { @@ -1257,7 +1202,7 @@ typedef struct { Eterm ref; Eterm ref_heap[ERTS_REF_THING_SIZE]; Uint req_sched; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; } ErtsSystemCheckReq; @@ -1289,10 +1234,8 @@ reply_sched_wall_time(void *vswtrp) ErlOffHeap *ohp = NULL; ErtsMessage *mp = NULL; - ASSERT(esdp); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + if (swtrp->set) { if (!swtrp->enable && esdp->sched_wall_time.enabled) { esdp->sched_wall_time.u.need = erts_sched_balance_util; @@ -1322,7 +1265,6 @@ reply_sched_wall_time(void *vswtrp) hpp = NULL; szp = &sz; -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp->sched_wall_time.enabled && swtrp->req_sched == esdp->no && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) { @@ -1404,7 +1346,6 @@ reply_sched_wall_time(void *vswtrp) erts_free(ERTS_ALC_T_TMP, dswt); } else -#endif { /* Reply with info about this scheduler only... */ @@ -1441,11 +1382,11 @@ reply_sched_wall_time(void *vswtrp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); - if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) + if (erts_atomic32_dec_read_nob(&swtrp->refc) == 0) swtreq_free(vswtrp); } @@ -1458,11 +1399,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable, ErtsSchedWallTimeReq *swtrp; Eterm *hp; + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + if (!set && !esdp->sched_wall_time.enabled) return THE_NON_VALUE; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif swtrp = swtreq_alloc(); ref = erts_make_ref(c_p); @@ -1473,22 +1413,18 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable, swtrp->proc = c_p; swtrp->ref = STORE_NC(&hp, NULL, ref); swtrp->req_sched = esdp->no; -#ifdef ERTS_DIRTY_SCHEDULERS swtrp->want_dirty_cpu = want_dirty_cpu; swtrp->want_dirty_io = want_dirty_io; -#endif - erts_smp_atomic32_init_nob(&swtrp->refc, + erts_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, reply_sched_wall_time, (void *) swtrp); -#endif reply_sched_wall_time((void *) swtrp); @@ -1509,10 +1445,7 @@ reply_system_check(void *vscrp) ErlOffHeap *ohp = NULL; ErtsMessage *mp = NULL; - ASSERT(esdp); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); sz = ERTS_REF_THING_SIZE; mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -1525,11 +1458,11 @@ reply_system_check(void *vscrp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); erts_proc_dec_refc(rp); - if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + if (erts_atomic32_dec_read_nob(&scrp->refc) == 0) screq_free(vscrp); } @@ -1547,17 +1480,15 @@ Eterm erts_system_check_request(Process *c_p) { scrp->proc = c_p; scrp->ref = STORE_NC(&hp, NULL, ref); scrp->req_sched = esdp->no; - erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + erts_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, reply_system_check, (void *) scrp); -#endif reply_system_check((void *) scrp); @@ -1595,6 +1526,12 @@ erts_proclist_create(Process *p) return proclist_create(p); } +ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + void erts_proclist_destroy(ErtsProcList *plp) { @@ -1612,7 +1549,7 @@ erts_psd_set_init(Process *p, int ix, void *data) for (i = 0; i < ERTS_PSD_SIZE; i++) new_psd->data[i] = NULL; - psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd, + psd = (ErtsPSD *) erts_atomic_cmpxchg_mb(&p->psd, (erts_aint_t) new_psd, (erts_aint_t) NULL); if (psd) @@ -1624,21 +1561,21 @@ erts_psd_set_init(Process *p, int ix, void *data) return old; } -#ifdef ERTS_SMP void -erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flags) { switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { case ERTS_SSI_FLG_POLL_SLEEPING: - erts_sys_schedule_interrupt(1); + erts_check_io_interrupt(ssi->psi, 1); break; case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING: /* * Thread progress blocking while poll sleeping; need * to signal on both... */ - erts_sys_schedule_interrupt(1); + erts_check_io_interrupt(ssi->psi, 1); /* fall through */ case ERTS_SSI_FLG_TSE_SLEEPING: erts_tse_set(ssi->event); @@ -1652,7 +1589,6 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) } } -#endif static ERTS_INLINE void set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, @@ -1668,11 +1604,7 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); if ((old_flgs & flgs) != flgs) { -#ifdef ERTS_SMP erts_sched_poke(ssi); -#else - erts_sys_schedule_interrupt(1); -#endif } } } @@ -1688,11 +1620,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); if ((old_flgs & flgs) != flgs) { -#ifdef ERTS_SMP erts_sched_poke(ssi); -#else - erts_sys_schedule_interrupt(1); -#endif } } @@ -1708,7 +1636,6 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } -#ifdef ERTS_SMP static ERTS_INLINE void haw_chk_later_cleanup_op_wakeup(ErtsAuxWorkData *awdp, ErtsThrPrgrVal val) @@ -1778,9 +1705,9 @@ static ERTS_INLINE void haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp) { ErtsThrPrgrVal current = awdp->current_thr_prgr; -#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + if (current != ERTS_THR_PRGR_INVALID && !erts_thr_progress_equal(current, erts_thr_progress_current())) { /* @@ -1797,9 +1724,7 @@ handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, in { int jix, max_jix; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif ASSERT(awdp->delayed_wakeup.next != ERTS_DELAYED_WAKEUP_INFINITY); @@ -1857,7 +1782,6 @@ schedule_aux_work_wakeup(ErtsAuxWorkData *awdp, } } -#endif typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { @@ -1898,11 +1822,7 @@ init_misc_aux_work(void) sizeof(erts_algnd_misc_aux_work_q_t) * (erts_no_schedulers+1)); -#ifdef ERTS_SMP ix = 0; /* aux_thread + schedulers */ -#else - ix = 1; /* scheduler only */ -#endif for (; ix <= erts_no_schedulers; ix++) { qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1); @@ -1920,10 +1840,8 @@ misc_aux_work_clean(ErtsThrQ_t *q, set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); return aux_work | ERTS_SSI_AUX_WORK_MISC; case ERTS_THR_Q_NEED_THR_PRGR: -#ifdef ERTS_SMP set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); haw_thr_prgr_soft_wakeup(awdp, erts_thr_q_need_thr_progress(q)); -#endif case ERTS_THR_Q_CLEAN: break; } @@ -1949,16 +1867,14 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC); } -#ifdef ERTS_SMP static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), awdp->misc.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; @@ -1970,7 +1886,6 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR); } -#endif static ERTS_INLINE void schedule_misc_aux_work(int sched_id, @@ -1980,11 +1895,7 @@ schedule_misc_aux_work(int sched_id, ErtsThrQ_t *q; erts_misc_aux_work_t *mawp; -#ifdef ERTS_SMP ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers); -#else - ASSERT(sched_id == 1); -#endif q = &misc_aux_work_queues[sched_id].q; mawp = misc_aux_work_alloc(); @@ -2010,12 +1921,13 @@ erts_schedule_multi_misc_aux_work(int ignore_self, int id, self = 0; if (ignore_self) { - ErtsSchedulerData *esdp = erts_get_scheduler_data(); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif - if (esdp) - self = (int) esdp->no; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + + /* ignore_self is meaningless on dirty schedulers since aux work can + * only run on normal schedulers, and their ids do not translate. */ + if(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + self = (int)esdp->no; + } } ASSERT(0 < max_sched && max_sched <= erts_no_schedulers); @@ -2027,7 +1939,6 @@ erts_schedule_multi_misc_aux_work(int ignore_self, } } -#if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *vno) @@ -2043,9 +1954,9 @@ handle_async_ready(ErtsAuxWorkData *awdp, int waiting) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; -#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); if (erts_check_async_ready(awdp->async_ready.queue)) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) @@ -2055,9 +1966,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, } return aux_work; } -#ifdef ERTS_SMP awdp->async_ready.need_thr_prgr = 0; -#endif set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY) | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); @@ -2070,10 +1979,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, { void *thr_prgr_p; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif -#ifdef ERTS_SMP + if (awdp->async_ready.need_thr_prgr && !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), awdp->async_ready.thr_prgr)) { @@ -2082,26 +1989,20 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, awdp->async_ready.need_thr_prgr = 0; thr_prgr_p = (void *) &awdp->async_ready.thr_prgr; -#else - thr_prgr_p = NULL; -#endif switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) { case ERTS_ASYNC_READY_CLEAN: unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#ifdef ERTS_SMP case ERTS_ASYNC_READY_NEED_THR_PRGR: haw_thr_prgr_soft_wakeup(awdp, awdp->async_ready.thr_prgr); awdp->async_ready.need_thr_prgr = 1; return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif default: return aux_work; } } -#endif /* ERTS_USE_ASYNC_READY_Q */ static ERTS_INLINE erts_aint32_t @@ -2110,9 +2011,8 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) ErtsSchedulerSleepInfo *ssi = awdp->ssi; erts_aint32_t res; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM @@ -2126,7 +2026,6 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) return aux_work; } -#ifdef ERTS_SMP void erts_alloc_notify_delayed_dealloc(int ix) @@ -2160,9 +2059,9 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; ERTS_MSACC_PUSH_STATE_M_X(); -#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, @@ -2199,9 +2098,8 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; @@ -2258,9 +2156,8 @@ handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); erts_handle_canceled_timers((void *) awdp->esdp, &need_thr_progress, @@ -2294,9 +2191,8 @@ handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; @@ -2339,9 +2235,8 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait int lops; ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif + for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) { ErtsThrPrgrLaterOp *lop = awdp->later_op.first; @@ -2371,7 +2266,7 @@ enqueue_later_op(ErtsSchedulerData *esdp, ErtsThrPrgrLaterOp *lop) { ErtsThrPrgrVal later = erts_thr_progress_later(esdp); - ASSERT(esdp); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); lop->func = later_func; lop->data = later_data; @@ -2387,20 +2282,15 @@ enqueue_later_op(ErtsSchedulerData *esdp, return later; } -#endif /* ERTS_SMP */ void erts_schedule_thr_prgr_later_op(void (*later_func)(void *), void *later_data, ErtsThrPrgrLaterOp *lop) { -#ifndef ERTS_SMP - later_func(later_data); -#else ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); haw_thr_prgr_wakeup(&esdp->aux_work_data, later); -#endif } void @@ -2409,13 +2299,9 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), ErtsThrPrgrLaterOp *lop, UWord size) { -#ifndef ERTS_SMP - later_func(later_data); -#else ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsThrPrgrVal later = enqueue_later_op(esdp, later_func, later_data, lop); haw_thr_prgr_later_cleanup_op_wakeup(&esdp->aux_work_data, later, size); -#endif } static ERTS_INLINE erts_aint32_t @@ -2424,9 +2310,7 @@ handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int w ErtsSchedulerSleepInfo *ssi = awdp->ssi; erts_aint32_t saved_aux_work, flags; -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#endif flags = awdp->debug.wait_completed.flags; @@ -2467,11 +2351,7 @@ setup_thr_debug_wait_completed(void *vproc) ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsAuxWorkData *awdp; erts_aint32_t wait_flags, aux_work_flags; -#ifdef ERTS_SMP awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data; -#else - awdp = &esdp->aux_work_data; -#endif wait_flags = 0; aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; @@ -2480,18 +2360,14 @@ setup_thr_debug_wait_completed(void *vproc) erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); wait_flags |= (ERTS_SSI_AUX_WORK_DD | ERTS_SSI_AUX_WORK_DD_THR_PRGR); -#ifdef ERTS_SMP aux_work_flags |= ERTS_SSI_AUX_WORK_DD; -#endif } if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); -#ifdef ERTS_SMP if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; -#endif } set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags); @@ -2510,21 +2386,17 @@ static void later_thr_debug_wait_completed(void *vlop) { struct debug_lop *lop = vlop; erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; -#ifdef ERTS_SMP count += 1; /* aux thread */ -#endif if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { /* scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, setup_thr_debug_wait_completed, lop->proc); -#ifdef ERTS_SMP /* aux_thread */ erts_schedule_misc_aux_work(0, setup_thr_debug_wait_completed, lop->proc); -#endif } erts_free(ERTS_ALC_T_DEBUG, lop); } @@ -2545,9 +2417,7 @@ erts_debug_wait_completed(Process *c_p, int flags) { /* Only one process at a time can do this */ erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); -#ifdef ERTS_SMP count += 1; /* aux thread */ -#endif if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { @@ -2576,7 +2446,7 @@ notify_reap_ports_relb(void) } } -erts_smp_atomic32_t erts_halt_progress; +erts_atomic32_t erts_halt_progress; int erts_halt_code; static ERTS_INLINE erts_aint32_t @@ -2585,9 +2455,9 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING); - if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { + if (erts_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { int i, max = erts_ptab_max(&erts_port); - erts_smp_atomic32_set_nob(&erts_halt_progress, 1); + erts_atomic32_set_nob(&erts_halt_progress, 1); for (i = 0; i < max; i++) { erts_aint32_t state; Port *prt = erts_pix2port(i); @@ -2600,21 +2470,21 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) /* We need to set the halt flag - get the port lock */ - erts_smp_port_lock(prt); + erts_port_lock(prt); state = erts_atomic32_read_nob(&prt->state); if (!(state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP | ERTS_PORT_SFLG_HALT))) { state = erts_atomic32_read_bor_relb(&prt->state, ERTS_PORT_SFLG_HALT); - erts_smp_atomic32_inc_nob(&erts_halt_progress); + erts_atomic32_inc_nob(&erts_halt_progress); if (!(state & (ERTS_PORT_SFLG_EXITING|ERTS_PORT_SFLG_CLOSING))) erts_deliver_port_exit(prt, prt->common.id, am_killed, 0, 1); } erts_port_release(prt); } - if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { + if (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0) { erts_flush_async_exit(erts_halt_code, ""); } } @@ -2675,7 +2545,6 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti #endif -#ifdef ERTS_SMP static ERTS_INLINE erts_aint32_t handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) @@ -2686,10 +2555,10 @@ handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin rq = awdp->esdp->run_queue; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); pnd_xtrs = rq->procs.pending_exiters; rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); if (erts_proclist_fetch(&pnd_xtrs, NULL)) do_handle_pending_exiters(pnd_xtrs); @@ -2697,7 +2566,6 @@ handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS; } -#endif static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) @@ -2729,9 +2597,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX); ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); -#ifdef ERTS_SMP haw_thr_prgr_current_reset(awdp); -#endif ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); ASSERT(aux_work); @@ -2750,7 +2616,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative * eachother. Most frequent first. */ -#ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP, handle_delayed_aux_work_wakeup); HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, @@ -2758,13 +2623,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) /* DD must be before DD_THR_PRGR */ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, handle_delayed_dealloc_thr_prgr); -#endif HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), handle_fix_alloc); -#ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP, handle_thr_prgr_later_op); HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS, @@ -2772,28 +2635,21 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR, handle_canceled_timers_thr_prgr); -#endif -#if ERTS_USE_ASYNC_READY_Q HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, handle_async_ready); /* ASYNC_READY must be before ASYNC_READY_CLEAN */ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, handle_async_ready_clean); -#endif -#ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, handle_misc_aux_work_thr_prgr); -#endif /* MISC_THR_PRGR must be before MISC */ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); -#ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS, handle_pending_exiters); -#endif HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -2819,10 +2675,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); -#ifdef ERTS_SMP if (waiting && !aux_work) haw_thr_prgr_current_check_progress(awdp); -#endif ERTS_MSACC_UPDATE_CACHE(); ERTS_MSACC_POP_STATE_M(); @@ -2921,11 +2775,7 @@ aux_work_timeout(void *vesdp) ASSERT(esdp == (ErtsSchedulerData *) vesdp); #endif -#ifdef ERTS_SMP i = 0; -#else - i = 1; -#endif for (; i <= erts_no_schedulers; i++) { erts_aint32_t type; @@ -2959,9 +2809,6 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) { erts_aint32_t old, refc; -#ifndef ERTS_SMP - ix = 1; -#endif ERTS_DBG_CHK_AUX_WORK_VAL(type); ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); @@ -2988,36 +2835,6 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) return old; } - - -static ERTS_INLINE void -sched_waiting_sys(Uint no, ErtsRunQueue *rq) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - ASSERT(rq->waiting >= 0); - (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); - rq->waiting++; - rq->waiting *= -1; - rq->woken = 0; - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(no), am_inactive); -} - -static ERTS_INLINE void -sched_active_sys(Uint no, ErtsRunQueue *rq) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif - ASSERT(rq->waiting < 0); - rq->waiting *= -1; - rq->waiting--; - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(no), am_active); -} - Uint erts_active_schedulers(void) { @@ -3028,64 +2845,10 @@ erts_active_schedulers(void) return as; } -#ifdef ERTS_SMP - -static ERTS_INLINE void -clear_sys_scheduling(void) -{ - erts_smp_atomic32_set_mb(&doing_sys_schedule, 0); -} - -static ERTS_INLINE int -try_set_sys_scheduling(void) -{ - return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); -} - -#endif - -static ERTS_INLINE int -prepare_for_sys_schedule(int non_blocking) -{ - if (non_blocking && erts_eager_check_io) { -#ifdef ERTS_SMP - return try_set_sys_scheduling(); -#else - return 1; -#endif - } - else { -#ifdef ERTS_SMP - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - clear_sys_scheduling(); - } - return 0; -#else - return !erts_port_task_have_outstanding_io_tasks(); -#endif - } -} - -#ifdef ERTS_SMP - -static ERTS_INLINE void -sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) -{ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif - ASSERT(rq->waiting < 0); - rq->waiting *= -1; -} - static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_SET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); if (rq->waiting < 0) @@ -3100,7 +2863,7 @@ sched_waiting(Uint no, ErtsRunQueue *rq) static ERTS_INLINE void sched_active(Uint no, ErtsRunQueue *rq) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); if (rq->waiting < 0) rq->waiting++; else @@ -3114,7 +2877,7 @@ empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) { if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && old_flags & ERTS_RUNQ_FLG_NONEMPTY) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); + erts_aint32_t empty = erts_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. @@ -3122,9 +2885,9 @@ empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif if (!erts_runq_supervision_interval) - erts_smp_atomic32_inc_relb(&no_empty_run_queues); + erts_atomic32_inc_relb(&no_empty_run_queues); else { - erts_smp_atomic32_inc_mb(&no_empty_run_queues); + erts_atomic32_inc_mb(&no_empty_run_queues); if (erts_atomic_read_nob(&runq_supervisor_sleeping)) ethr_event_set(&runq_supervision_event); } @@ -3154,7 +2917,7 @@ non_empty_runq(ErtsRunQueue *rq) Uint32 old_flags = ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_NONEMPTY); if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && (!(old_flags & ERTS_RUNQ_FLG_NONEMPTY))) { #ifdef DEBUG - erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues); + erts_aint32_t empty = erts_atomic32_read_nob(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. @@ -3162,10 +2925,10 @@ non_empty_runq(ErtsRunQueue *rq) ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif if (!erts_runq_supervision_interval) - erts_smp_atomic32_dec_relb(&no_empty_run_queues); + erts_atomic32_dec_relb(&no_empty_run_queues); else { erts_aint32_t no; - no = erts_smp_atomic32_dec_read_mb(&no_empty_run_queues); + no = erts_atomic32_dec_read_mb(&no_empty_run_queues); if (no > 0 && erts_atomic_read_nob(&runq_supervisor_sleeping)) ethr_event_set(&runq_supervision_event); } @@ -3194,7 +2957,7 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) do { nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC); nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; - oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -3211,7 +2974,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; do { - oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -3228,7 +2991,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + flgs = erts_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { break; @@ -3253,11 +3016,11 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) erts_tse_reset(ssi->event); else { ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING); - erts_sys_schedule_interrupt(0); + erts_check_io_interrupt(ssi->psi, 0); } while (1) { - oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) @@ -3284,7 +3047,7 @@ static void thr_prgr_prep_wait(void *vssi) { ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; - erts_smp_atomic32_read_bor_acqb(&ssi->flags, + erts_atomic32_read_bor_acqb(&ssi->flags, ERTS_SSI_FLG_SLEEPING); } @@ -3299,7 +3062,7 @@ thr_prgr_wait(void *vssi) while (1) { erts_aint32_t aflgs, nflgs; nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING; - aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + aflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (aflgs == xflgs) { erts_tse_wait(ssi->event); break; @@ -3314,13 +3077,19 @@ static void thr_prgr_fin_wait(void *vssi) { ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; - erts_smp_atomic32_read_band_nob(&ssi->flags, + erts_atomic32_read_band_nob(&ssi->flags, ~(ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_TSE_SLEEPING)); } static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_aux_thread_poke() +{ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -3329,6 +3098,7 @@ aux_thread(void *unused) erts_aint32_t aux_work; ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; + ERTS_MSACC_DECLARE_CACHE(); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -3337,6 +3107,7 @@ aux_thread(void *unused) } #endif + erts_port_task_pre_alloc_init_thread(); ssi->event = erts_tse_fetch(); erts_msacc_init_thread("aux", 1, 1); @@ -3351,9 +3122,14 @@ aux_thread(void *unused) init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; +#if ERTS_POLL_USE_FALLBACK + ssi->psi = erts_create_pollset_thread(-1); +#endif sched_prep_spin_wait(ssi); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); + while (1) { erts_aint32_t flgs; @@ -3362,30 +3138,54 @@ aux_thread(void *unused) if (!thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 1); aux_work = handle_aux_work(awdp, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); if (aux_work && erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); } if (!aux_work) { + +#ifdef ERTS_BREAK_REQUESTED + if (ERTS_BREAK_REQUESTED) + erts_do_break_handling(); +#endif + if (thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 0); - erts_thr_progress_prepare_wait(NULL); + +#if ERTS_POLL_USE_FALLBACK flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_check_io(ssi->psi); + } + } +#else + erts_thr_progress_prepare_wait(NULL); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; + int res; ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); } - } - erts_thr_progress_finalize_wait(NULL); + } + erts_thr_progress_finalize_wait(NULL); +#endif } flgs = sched_prep_spin_wait(ssi); @@ -3393,9 +3193,79 @@ aux_thread(void *unused) return NULL; } -static void suspend_scheduler(ErtsSchedulerData *esdp); +static void * +poll_thread(void *arg) +{ + int id = (int)(UWord)arg; + ErtsAuxWorkData *awdp = poll_thread_aux_work_data+id; + ErtsSchedulerSleepInfo *ssi = ERTS_POLL_THREAD_SLEEP_INFO_IX(id); + erts_aint32_t aux_work; + ErtsThrPrgrCallbacks callbacks; + int thr_prgr_active = 1; + struct erts_poll_thread *psi = erts_create_pollset_thread(id); + ERTS_MSACC_DECLARE_CACHE(); + +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[] = "poll_thread"; + erts_lc_set_thread_name(buf); + } +#endif + + erts_port_task_pre_alloc_init_thread(); + ssi->event = erts_tse_fetch(); + + erts_msacc_init_thread("poll", id, 0); -#endif /* ERTS_SMP */ + callbacks.arg = (void *) ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 0); + init_aux_work_data(awdp, NULL, NULL); + awdp->ssi = ssi; + ssi->psi = psi; + + sched_prep_spin_wait(ssi); + + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); + + while (1) { + erts_aint32_t flgs; + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + aux_work = handle_aux_work(awdp, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + } + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_check_io(psi); + } + } + } + + flgs = sched_prep_spin_wait(ssi); + } + return NULL; +} static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) @@ -3404,386 +3274,168 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; erts_aint32_t aux_work = 0; -#ifdef ERTS_SMP int thr_prgr_active = 1; erts_aint32_t flgs; -#endif ERTS_MSACC_PUSH_STATE_M(); -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - erts_smp_spin_lock(&rq->sleepers.lock); -#endif + erts_spin_lock(&rq->sleepers.lock); flgs = sched_prep_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_SUSPENDED) { /* Go suspend instead... */ -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - erts_smp_spin_unlock(&rq->sleepers.lock); -#endif + erts_spin_unlock(&rq->sleepers.lock); return; } -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { ssi->prev = NULL; ssi->next = rq->sleepers.list; if (rq->sleepers.list) rq->sleepers.list->prev = ssi; rq->sleepers.list = ssi; - erts_smp_spin_unlock(&rq->sleepers.lock); + erts_spin_unlock(&rq->sleepers.lock); dirty_active(esdp, -1); } -#endif - - /* - * If all schedulers are waiting, one of them *should* - * be waiting in erl_sys_schedule() - */ - - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { - - sched_waiting(esdp->no, rq); - - erts_smp_runq_unlock(rq); - - spincount = sched_busy_wait.tse; - tse_wait: + sched_waiting(esdp->no, rq); - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - dirty_sched_wall_time_change(esdp, working = 0); - else if (thr_prgr_active != working) - sched_wall_time_change(esdp, working = thr_prgr_active); + erts_runq_unlock(rq); - while (1) { - ErtsMonotonicTime current_time = 0; - - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - ERTS_MSACC_UPDATE_CACHE(); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); - } + spincount = sched_busy_wait.tse; - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - erts_bump_timers(esdp->timer_wheel, current_time); - } - } - } - else { - ErtsMonotonicTime timeout_time; - int do_timeout = 0; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - do_timeout = (current_time >= timeout_time); - } else { - current_time = 0; - timeout_time = ERTS_MONOTONIC_TIME_MAX; - } - if (do_timeout) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - else { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); - } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 0); + else if (thr_prgr_active != working) + sched_wall_time_change(esdp, working = thr_prgr_active); - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); - do { - Sint64 timeout; - if (current_time >= timeout_time) - break; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; - } else - timeout = -1; - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - res = erts_tse_twait(ssi->event, timeout); - ERTS_MSACC_POP_STATE_M(); - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); - } while (res == EINTR); - } - } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - erts_thr_progress_finalize_wait(esdp); - } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) - erts_bump_timers(esdp->timer_wheel, current_time); - } + while (1) { + ErtsMonotonicTime current_time = 0; - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - break; - } + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } - flgs = sched_prep_cont_spin_wait(ssi); - spincount = sched_busy_wait.aux_work; + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + } + else { + ErtsMonotonicTime timeout_time; + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else { + current_time = 0; + timeout_time = ERTS_MONOTONIC_TIME_MAX; + } + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - break; - } + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + res = erts_tse_twait(ssi->event, timeout); + ERTS_MSACC_POP_STATE_M(); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } + } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); + } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); + } - } + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } - if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) - erts_smp_atomic32_read_band_nob(&ssi->flags, - (ERTS_SSI_FLG_SUSPENDED - | ERTS_SSI_FLG_MSB_EXEC)); + flgs = sched_prep_cont_spin_wait(ssi); + spincount = sched_busy_wait.aux_work; - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - dirty_sched_wall_time_change(esdp, working = 1); - else if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; } - erts_smp_runq_lock(rq); - sched_active(esdp->no, rq); - } - else -#endif - { - - erts_smp_atomic32_set_relb(&function_calls, 0); - *fcalls = 0; - -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif - sched_waiting_sys(esdp->no, rq); - - erts_smp_runq_unlock(rq); - - ASSERT(working); - sched_wall_time_change(esdp, working = 0); - - spincount = sched_busy_wait.sys_schedule; - if (spincount == 0) - goto sys_aux_work; - - while (spincount-- > 0) { - ErtsMonotonicTime current_time; - - sys_poll_aux_work: - - if (working) - sched_wall_time_change(esdp, working = 0); - - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - LTTNG2(scheduler_poll, esdp->no, 1); - erl_sys_schedule(1); /* Might give us something to do */ - - ERTS_MSACC_POP_STATE_M(); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - } - - sys_aux_work: -#ifndef ERTS_SMP - erts_sys_schedule_interrupt(0); -#endif - - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!working) - sched_wall_time_change(esdp, working = 1); -#ifdef ERTS_SMP - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); -#endif - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - ERTS_MSACC_UPDATE_CACHE(); -#ifdef ERTS_SMP - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); -#endif - } - -#ifndef ERTS_SMP - if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start) - goto sys_woken; -#else - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - - /* - * If we got new I/O tasks we aren't allowed to - * call erl_sys_schedule() until it is handled. - */ - if (erts_port_task_have_outstanding_io_tasks()) { - clear_sys_scheduling(); - /* - * Got to check that we still got I/O tasks; otherwise - * we have to continue checking for I/O... - */ - if (!prepare_for_sys_schedule(0)) { - spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; - goto tse_wait; - } - } -#endif - } - - erts_smp_runq_lock(rq); - -#ifdef ERTS_SMP - /* - * If we got new I/O tasks we aren't allowed to - * sleep in erl_sys_schedule(). - */ - if (erts_port_task_have_outstanding_io_tasks()) { - clear_sys_scheduling(); - - /* - * Got to check that we still got I/O tasks; otherwise - * we have to wait in erl_sys_schedule() after all... - */ - if (!prepare_for_sys_schedule(0)) { - /* - * Not allowed to wait in erl_sys_schedule; - * do tse wait instead... - */ - sched_change_waiting_sys_to_waiting(esdp->no, rq); - erts_smp_runq_unlock(rq); - spincount = 0; - goto tse_wait; - } - } -#endif - if (aux_work) { - erts_smp_runq_unlock(rq); - goto sys_poll_aux_work; - } -#ifdef ERTS_SMP - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); - if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_locked_woken; - } - erts_smp_runq_unlock(rq); - flgs = sched_prep_cont_spin_wait(ssi); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - goto sys_poll_aux_work; - } - - ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); -#endif - - erts_smp_runq_unlock(rq); - - if (working) - sched_wall_time_change(esdp, working = 0); -#ifdef ERTS_SMP - if (thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 0); -#endif - - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - LTTNG2(scheduler_poll, esdp->no, 0); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); - erl_sys_schedule(0); - - ERTS_MSACC_POP_STATE_M(); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - } - -#ifndef ERTS_SMP - if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start) - goto sys_aux_work; - sys_woken: -#else - flgs = sched_prep_cont_spin_wait(ssi); - if (flgs & ERTS_SSI_FLG_WAITING) - goto sys_aux_work; - - sys_woken: - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); - erts_smp_runq_lock(rq); - sys_locked_woken: - if (!thr_prgr_active) { - erts_smp_runq_unlock(rq); - erts_thr_progress_active(esdp, thr_prgr_active = 1); - erts_smp_runq_lock(rq); - } - clear_sys_scheduling(); - if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) - erts_smp_atomic32_read_band_nob(&ssi->flags, - (ERTS_SSI_FLG_SUSPENDED - | ERTS_SSI_FLG_MSB_EXEC)); -#endif - if (!working) - sched_wall_time_change(esdp, working = 1); - sched_active_sys(esdp->no, rq); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 1); + else if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } + erts_runq_lock(rq); + sched_active(esdp->no, rq); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) dirty_active(esdp, 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); } -#ifdef ERTS_SMP static ERTS_INLINE erts_aint32_t ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) @@ -3793,7 +3445,7 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) erts_aint32_t nflgs = 0; erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; while (1) { - oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); @@ -3807,13 +3459,12 @@ ssi_wake(ErtsSchedulerSleepInfo *ssi) erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi)); } -#ifdef ERTS_DIRTY_SCHEDULERS static void dcpu_sched_ix_suspend_wake(Uint ix) { ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); ssi_wake(ssi); } @@ -3821,7 +3472,7 @@ static void dio_sched_ix_suspend_wake(Uint ix) { ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); ssi_wake(ssi); } @@ -3839,7 +3490,6 @@ dio_sched_ix_wake(Uint ix) } #endif -#endif static void wake_scheduler(ErtsRunQueue *rq) @@ -3852,13 +3502,12 @@ wake_scheduler(ErtsRunQueue *rq) * so all code *should* handle this without having * the lock on the run queue. */ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) + ERTS_LC_ASSERT(!erts_lc_runq_is_locked(rq) || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); ssi_wake(rq->scheduler->ssi); } -#ifdef ERTS_DIRTY_SCHEDULERS static void wake_dirty_schedulers(ErtsRunQueue *rq, int one) { @@ -3868,10 +3517,10 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one) ASSERT(ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); sl = &rq->sleepers; - erts_smp_spin_lock(&sl->lock); + erts_spin_lock(&sl->lock); ssi = sl->list; if (!ssi) { - erts_smp_spin_unlock(&sl->lock); + erts_spin_unlock(&sl->lock); if (one) wake_scheduler(rq); } else if (one) { @@ -3885,14 +3534,14 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one) if (ssi->next) ssi->next->prev = ssi->prev; - erts_smp_spin_unlock(&sl->lock); + erts_spin_unlock(&sl->lock); ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); } else { sl->list = NULL; - erts_smp_spin_unlock(&sl->lock); + erts_spin_unlock(&sl->lock); ERTS_THR_MEMORY_BARRIER; do { @@ -3909,7 +3558,6 @@ wake_dirty_scheduler(ErtsRunQueue *rq) wake_dirty_schedulers(rq, 1); } -#endif #define ERTS_NO_USED_RUNQS_SHIFT 16 #define ERTS_NO_RUNQS_MASK 0xffffU @@ -3923,13 +3571,13 @@ init_no_runqs(int active, int used) { erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK); no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT); - erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs); + erts_atomic32_init_nob(&balance_info.no_runqs, no_runqs); } static ERTS_INLINE void get_no_runqs(int *active, int *used) { - erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs); + erts_aint32_t no_runqs = erts_atomic32_read_nob(&balance_info.no_runqs); if (active) *active = (int) (no_runqs & ERTS_NO_RUNQS_MASK); if (used) @@ -3939,12 +3587,12 @@ get_no_runqs(int *active, int *used) static ERTS_INLINE void set_no_used_runqs(int used) { - erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); + erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT; new |= exp & ERTS_NO_RUNQS_MASK; - act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); + act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -3954,14 +3602,14 @@ set_no_used_runqs(int used) static ERTS_INLINE void set_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); + erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs); while (1) { erts_aint32_t act, new; if ((exp & ERTS_NO_RUNQS_MASK) == active) break; new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); new |= active & ERTS_NO_RUNQS_MASK; - act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); + act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) break; exp = act; @@ -3971,14 +3619,14 @@ set_no_active_runqs(int active) static ERTS_INLINE int try_inc_no_active_runqs(int active) { - erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs); + erts_aint32_t exp = erts_atomic32_read_nob(&balance_info.no_runqs); if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active) return 0; if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) { erts_aint32_t new, act; new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT); new |= active & ERTS_NO_RUNQS_MASK; - act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); + act = erts_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp); if (act == exp) return 1; } @@ -4040,25 +3688,20 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) } } -#endif /* ERTS_SMP */ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { -#ifdef ERTS_SMP if (runq) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) wake_dirty_scheduler(runq); else -#endif wake_scheduler(runq); } -#endif } void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } @@ -4066,16 +3709,12 @@ erts_smp_notify_inc_runq(ErtsRunQueue *runq) void erts_sched_notify_check_cpu_bind(void) { -#ifdef ERTS_SMP int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); wake_scheduler(rq); } -#else - erts_sched_check_cpu_bind(erts_get_scheduler_data()); -#endif } @@ -4084,9 +3723,9 @@ enqueue_process(ErtsRunQueue *runq, int prio, Process *p) { ErtsRunPrioQueue *rpq; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); - erts_smp_inc_runq_len(runq, &runq->procs.prio_info[prio], prio); + erts_inc_runq_len(runq, &runq->procs.prio_info[prio], prio); if (prio == PRIORITY_LOW) { p->schedule_count = RESCHEDULE_LOW; @@ -4114,7 +3753,7 @@ unqueue_process(ErtsRunQueue *runq, Process *prev_proc, Process *proc) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); if (prev_proc) prev_proc->next = proc->next; @@ -4126,7 +3765,7 @@ unqueue_process(ErtsRunQueue *runq, if (!rpq->first) rpq->last = NULL; - erts_smp_dec_runq_len(runq, rqi, prio); + erts_dec_runq_len(runq, rqi, prio); } @@ -4139,7 +3778,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) ErtsRunQueueInfo *rqi; Process *p; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(runq)); ASSERT(PRIORITY_NORMAL == prio_q || PRIORITY_HIGH == prio_q @@ -4150,9 +3789,9 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) if (!p) return NULL; - ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); if (statep) *statep = state; @@ -4185,11 +3824,10 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q) static ERTS_INLINE void free_proxy_proc(Process *proxy) { - ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); erts_free(ERTS_ALC_T_PROC, proxy); } -#ifdef ERTS_SMP static ErtsRunQueue * check_immigration_need(ErtsRunQueue *c_rq, ErtsMigrationPath *mp, int prio) @@ -4242,7 +3880,7 @@ static void immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) { Uint32 iflags, iflag; - erts_smp_runq_unlock(c_rq); + erts_runq_unlock(c_rq); ASSERT(erts_thr_progress_is_managed_thread()); @@ -4283,13 +3921,13 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = check_immigration_need(c_rq, mp, prio); if (rq) { - erts_smp_runq_lock(rq); + erts_runq_lock(rq); if (prio == ERTS_PORT_PRIO_LEVEL) { Port *prt; prt = erts_dequeue_port(rq); if (prt) RUNQ_SET_RQ(&prt->run_queue, c_rq); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); if (prt) { /* port might terminate while we have no lock... */ rq = erts_port_runq(prt); @@ -4301,7 +3939,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) erts_enqueue_port(c_rq, prt); if (!iflag) return; /* done */ - erts_smp_runq_unlock(c_rq); + erts_runq_unlock(c_rq); } } } @@ -4315,38 +3953,38 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) while (proc) { erts_aint32_t state; - state = erts_smp_atomic32_read_acqb(&proc->state); + state = erts_atomic32_read_acqb(&proc->state); if (!(ERTS_PSFLG_BOUND & state) && (prio == (int) ERTS_PSFLGS_GET_PRQ_PRIO(state))) { ErtsRunQueueInfo *rqi = &rq->procs.prio_info[prio]; unqueue_process(rq, rpq, rqi, prio, prev_proc, proc); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); RUNQ_SET_RQ(&proc->run_queue, c_rq); rq_locked = 0; - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); enqueue_process(c_rq, prio, proc); if (!iflag) return; /* done */ - erts_smp_runq_unlock(c_rq); + erts_runq_unlock(c_rq); break; } prev_proc = proc; proc = proc->next; } if (rq_locked) - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } } } - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); } static ERTS_INLINE void suspend_run_queue(ErtsRunQueue *rq) { - erts_smp_atomic32_read_bor_nob(&rq->scheduler->ssi->flags, + erts_atomic32_read_bor_nob(&rq->scheduler->ssi->flags, ERTS_SSI_FLG_SUSPENDED); (void) ERTS_RUNQ_FLGS_SET(rq, ERTS_RUNQ_FLG_SUSPENDED); @@ -4363,7 +4001,7 @@ resume_run_queue(ErtsRunQueue *rq) ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK @@ -4378,19 +4016,19 @@ resume_run_queue(ErtsRunQueue *rq) rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len); + len = erts_atomic32_read_dirty(&rq->procs.prio_info[pix].len); rq->procs.prio_info[pix].max_len = len; rq->procs.prio_info[pix].reds = 0; } - len = erts_smp_atomic32_read_dirty(&rq->ports.info.len); + len = erts_atomic32_read_dirty(&rq->ports.info.len); rq->ports.info.max_len = len; rq->ports.info.reds = 0; - len = erts_smp_atomic32_read_dirty(&rq->len); + len = erts_atomic32_read_dirty(&rq->len); rq->max_len = len; } - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); nrml_sched_ix_resume_wake(rq->ix); } @@ -4405,18 +4043,17 @@ schedule_bound_processes(ErtsRunQueue *rq, ErtsStuckBoundProcesses *sbpp) { Process *proc, *next; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); proc = sbpp->first; while (proc) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); next = proc->next; enqueue_process(rq, (int) ERTS_PSFLGS_GET_PRQ_PRIO(state), proc); proc = next; } } -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE void clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) @@ -4436,11 +4073,10 @@ clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) #else (void) #endif - erts_smp_atomic32_read_band_mb(&p->dirty_state, ~qb); + erts_atomic32_read_band_mb(&p->dirty_state, ~qb); ASSERT(old & qb); } -#endif /* ERTS_DIRTY_SCHEDULERS */ static void @@ -4452,7 +4088,7 @@ evacuate_run_queue(ErtsRunQueue *rq, ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); @@ -4475,9 +4111,9 @@ evacuate_run_queue(ErtsRunQueue *rq, rq->misc.start = NULL; rq->misc.end = NULL; ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); - erts_smp_runq_lock(to_rq); + erts_runq_lock(to_rq); if (to_rq->misc.end) to_rq->misc.end->next = start; else @@ -4487,9 +4123,9 @@ evacuate_run_queue(ErtsRunQueue *rq, non_empty_runq(to_rq); - erts_smp_runq_unlock(to_rq); + erts_runq_unlock(to_rq); smp_notify_inc_runq(to_rq); - erts_smp_runq_lock(to_rq); + erts_runq_lock(to_rq); } if (rq->ports.start) { @@ -4505,7 +4141,7 @@ evacuate_run_queue(ErtsRunQueue *rq, ErtsRunQueue *prt_rq; prt = erts_dequeue_port(rq); RUNQ_SET_RQ(&prt->run_queue, to_rq); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); /* * The port might terminate while * we have no lock on it... @@ -4517,9 +4153,9 @@ evacuate_run_queue(ErtsRunQueue *rq, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); - erts_smp_runq_unlock(to_rq); + erts_runq_unlock(to_rq); } - erts_smp_runq_lock(rq); + erts_runq_lock(rq); prt = rq->ports.start; } smp_notify_inc_runq(to_rq); @@ -4556,7 +4192,7 @@ evacuate_run_queue(ErtsRunQueue *rq, free_proxy_proc(proc); goto handle_next_proc; } - real_state = erts_smp_atomic32_read_acqb(&real_proc->state); + real_state = erts_atomic32_read_acqb(&real_proc->state); } max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); @@ -4582,7 +4218,7 @@ evacuate_run_queue(ErtsRunQueue *rq, #else (void) #endif - erts_smp_atomic32_read_band_mb(&proc->state, + erts_atomic32_read_band_mb(&proc->state, ~clr_bits); ASSERT((old & clr_bits) == clr_bits); @@ -4602,17 +4238,17 @@ evacuate_run_queue(ErtsRunQueue *rq, } else { int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); to_rq = mp->prio[prio].runq; RUNQ_SET_RQ(&proc->run_queue, to_rq); - erts_smp_runq_lock(to_rq); + erts_runq_lock(to_rq); enqueue_process(to_rq, prio, proc); - erts_smp_runq_unlock(to_rq); + erts_runq_unlock(to_rq); notify = 1; - erts_smp_runq_lock(rq); + erts_runq_lock(rq); } handle_next_proc: @@ -4631,13 +4267,13 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, ErtsRunPrioQueue *rpq; if (*rq_lockedp) { - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); *rq_lockedp = 0; } - ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(!erts_lc_runq_is_locked(rq)); - erts_smp_runq_lock(vrq); + erts_runq_lock(vrq); if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING) goto no_procs; @@ -4673,16 +4309,16 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, proc = rpq->first; while (proc) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&proc->state); + erts_aint32_t state = erts_atomic32_read_acqb(&proc->state); if (!(ERTS_PSFLG_BOUND & state)) { /* Steal process */ int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); ErtsRunQueueInfo *rqi = &vrq->procs.prio_info[prio]; unqueue_process(vrq, rpq, rqi, prio, prev_proc, proc); - erts_smp_runq_unlock(vrq); + erts_runq_unlock(vrq); RUNQ_SET_RQ(&proc->run_queue, rq); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); *rq_lockedp = 1; enqueue_process(rq, prio, proc); return !0; @@ -4696,7 +4332,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, no_procs: - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(vrq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(vrq)); /* * Check for a runnable port to steal... @@ -4706,7 +4342,7 @@ no_procs: ErtsRunQueue *prt_rq; Port *prt = erts_dequeue_port(vrq); RUNQ_SET_RQ(&prt->run_queue, rq); - erts_smp_runq_unlock(vrq); + erts_runq_unlock(vrq); /* * The port might terminate while @@ -4727,7 +4363,7 @@ no_procs: } } - erts_smp_runq_unlock(vrq); + erts_runq_unlock(vrq); return 0; } @@ -4759,7 +4395,7 @@ try_steal_task(ErtsRunQueue *rq) res = 0; rq_locked = 1; - ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); + ERTS_LC_CHK_RUNQ_LOCK(rq, rq_locked); get_no_runqs(&active_rqs, &blnc_rqs); @@ -4772,7 +4408,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { + while (erts_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -4787,7 +4423,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { + while (erts_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -4804,7 +4440,7 @@ try_steal_task(ErtsRunQueue *rq) done: if (!rq_locked) - erts_smp_runq_lock(rq); + erts_runq_lock(rq); if (res) return res; @@ -4930,7 +4566,7 @@ alloc_mpaths(void) { void *block; ErtsMigrationPaths *res; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&balance_info.update_mtx)); res = mpaths.freelist; if (res) { @@ -4953,7 +4589,7 @@ retire_mpaths(ErtsMigrationPaths *mps) { ErtsThrPrgrVal current; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&balance_info.update_mtx)); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&balance_info.update_mtx)); current = erts_thr_progress_current(); @@ -4999,7 +4635,7 @@ check_balance(ErtsRunQueue *c_rq) int sched_util_balancing; #endif - if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) { + if (erts_atomic32_xchg_nob(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } @@ -5007,15 +4643,15 @@ check_balance(ErtsRunQueue *c_rq) get_no_runqs(NULL, &blnc_no_rqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + erts_atomic32_set_nob(&balance_info.checking_balance, 0); return; } - erts_smp_runq_unlock(c_rq); + erts_runq_unlock(c_rq); if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + erts_atomic32_set_nob(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -5025,7 +4661,7 @@ check_balance(ErtsRunQueue *c_rq) rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; }); - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); return; } @@ -5038,7 +4674,7 @@ check_balance(ErtsRunQueue *c_rq) * is manipulated. Such updates of the migration information * might clash with balancing. */ - erts_smp_mtx_lock(&balance_info.update_mtx); + erts_mtx_lock(&balance_info.update_mtx); forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; @@ -5046,10 +4682,10 @@ check_balance(ErtsRunQueue *c_rq) get_no_runqs(¤t_active, &blnc_no_rqs); if (blnc_no_rqs == 1) { - erts_smp_mtx_unlock(&balance_info.update_mtx); - erts_smp_runq_lock(c_rq); + erts_mtx_unlock(&balance_info.update_mtx); + erts_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + erts_atomic32_set_nob(&balance_info.checking_balance, 0); return; } @@ -5065,7 +4701,7 @@ check_balance(ErtsRunQueue *c_rq) /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); run_queue_info[qix].flags = ERTS_RUNQ_FLGS_GET_NOB(rq); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { @@ -5093,7 +4729,7 @@ check_balance(ErtsRunQueue *c_rq) run_queue_info[qix].sched_util = erts_get_sched_util(rq, 1, 0); #endif - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } full_scheds = 0; @@ -5532,7 +5168,7 @@ erts_fprintf(stderr, "--------------------------------\n"); Uint32 flags = run_queue_info[qix].flags; ErtsRunQueue *rq = ERTS_RUNQ_IX(qix); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); ASSERT(!(flags & ERTS_RUNQ_FLG_OUT_OF_WORK)); if (rq->waiting) flags |= ERTS_RUNQ_FLG_OUT_OF_WORK; @@ -5547,27 +5183,27 @@ erts_fprintf(stderr, "--------------------------------\n"); rq->out_of_work_count = 0; (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); - rq->max_len = erts_smp_atomic32_read_dirty(&rq->len); + rq->max_len = erts_atomic32_read_dirty(&rq->len); for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { ErtsRunQueueInfo *rqi; rqi = (pix == ERTS_PORT_PRIO_LEVEL ? &rq->ports.info : &rq->procs.prio_info[pix]); - erts_smp_reset_max_len(rq, rqi); + erts_reset_max_len(rq, rqi); rqi->reds = 0; } rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } - erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0); + erts_atomic32_set_nob(&balance_info.checking_balance, 0); balance_info.n++; retire_mpaths(old_mpaths); - erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_mtx_unlock(&balance_info.update_mtx); - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); } static void @@ -5575,7 +5211,7 @@ change_no_used_runqs(int used) { ErtsMigrationPaths *new_mpaths, *old_mpaths; int qix; - erts_smp_mtx_lock(&balance_info.update_mtx); + erts_mtx_lock(&balance_info.update_mtx); set_no_used_runqs(used); old_mpaths = erts_get_migration_paths_managed(); @@ -5622,28 +5258,23 @@ change_no_used_runqs(int used) /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; - erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_mtx_unlock(&balance_info.update_mtx); - erts_smp_runq_lock(ERTS_RUNQ_IX(0)); + erts_runq_lock(ERTS_RUNQ_IX(0)); ERTS_RUNQ_IX(0)->check_balance_reds = 0; - erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); + erts_runq_unlock(ERTS_RUNQ_IX(0)); } -#endif /* #ifdef ERTS_SMP */ Uint erts_debug_nbalance(void) { -#ifdef ERTS_SMP Uint n; - erts_smp_mtx_lock(&balance_info.update_mtx); + erts_mtx_lock(&balance_info.update_mtx); n = balance_info.n; - erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_mtx_unlock(&balance_info.update_mtx); return n; -#else - return 0; -#endif } /* Wakeup other schedulers */ @@ -5689,7 +5320,6 @@ typedef enum { #define ERTS_WAKEUP_OTHER_DEC_LEGACY 10 #define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10) -#ifdef ERTS_SMP static struct { ErtsSchedWakeupOtherThreshold threshold; @@ -5705,7 +5335,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1; + int left_len = erts_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { int wo_reduce = wo_reds << wakeup_other.dec_shift; wo_reduce &= wakeup_other.dec_mask; @@ -5717,16 +5347,14 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) rq->wakeup_other += (left_len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { wake_dirty_scheduler(rq); } } else -#endif { int empty_rqs = - erts_smp_atomic32_read_acqb(&no_empty_run_queues); + erts_atomic32_read_acqb(&no_empty_run_queues); if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); if (empty_rqs != 0) @@ -5778,7 +5406,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len); + erts_aint32_t len = erts_atomic32_read_dirty(&rq->len); if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) @@ -5789,7 +5417,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) else { if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); - if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { + if (erts_atomic32_read_acqb(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -5840,7 +5468,7 @@ static int no_runqs_to_supervise(void) { int used; - erts_aint32_t nerq = erts_smp_atomic32_read_acqb(&no_empty_run_queues); + erts_aint32_t nerq = erts_atomic32_read_acqb(&no_empty_run_queues); if (nerq <= 0) return 0; get_no_runqs(NULL, &used); @@ -5873,26 +5501,23 @@ runq_supervisor(void *unused) for (ix = 0; ix < no_rqs; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { - erts_smp_runq_lock(rq); - if (erts_smp_atomic32_read_dirty(&rq->len) != 0) + erts_runq_lock(rq); + if (erts_atomic32_read_dirty(&rq->len) != 0) wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } } } return NULL; } -#endif void erts_early_init_scheduling(int no_schedulers) { aux_work_timeout_early_init(no_schedulers); -#ifdef ERTS_SMP wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; -#endif sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); @@ -5903,7 +5528,6 @@ erts_early_init_scheduling(int no_schedulers) int erts_sched_set_wakeup_other_thresold(char *str) { -#ifdef ERTS_SMP ErtsSchedWakeupOtherThreshold threshold; if (sys_strcmp(str, "very_high") == 0) threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH; @@ -5920,20 +5544,11 @@ erts_sched_set_wakeup_other_thresold(char *str) wakeup_other.threshold = threshold; set_wakeup_other_data(); return 0; -#else - if (sys_strcmp(str, "very_high") == 0 || sys_strcmp(str, "high") == 0 || - sys_strcmp(str, "medium") == 0 || sys_strcmp(str, "low") == 0 || - sys_strcmp(str, "very_low") == 0) { - return 0; - } - return EINVAL; -#endif } int erts_sched_set_wakeup_other_type(char *str) { -#ifdef ERTS_SMP ErtsSchedWakeupOtherType type; if (sys_strcmp(str, "default") == 0) type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; @@ -5943,12 +5558,6 @@ erts_sched_set_wakeup_other_type(char *str) return EINVAL; wakeup_other.type = type; return 0; -#else - if (sys_strcmp(str, "default") == 0 || sys_strcmp(str, "legacy") == 0) { - return 0; - } - return EINVAL; -#endif } int @@ -6019,7 +5628,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) case ERTS_SCHED_NORMAL: id = (int) esdp->no; break; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: id = (int) erts_no_schedulers; id += (int) esdp->dirty_no; @@ -6029,7 +5637,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) id += (int) erts_no_dirty_cpu_schedulers; id += (int) esdp->dirty_no; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); break; @@ -6039,7 +5646,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->sched_id = id; awdp->esdp = esdp; awdp->ssi = esdp ? esdp->ssi : NULL; -#ifdef ERTS_SMP awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; @@ -6048,15 +5654,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->later_op.size = 0; awdp->later_op.first = NULL; awdp->later_op.last = NULL; -#endif -#ifdef ERTS_USE_ASYNC_READY_Q -#ifdef ERTS_SMP awdp->async_ready.need_thr_prgr = 0; awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; -#endif awdp->async_ready.queue = NULL; -#endif -#ifdef ERTS_SMP awdp->delayed_wakeup.next = ERTS_DELAYED_WAKEUP_INFINITY; if (!dawwp) { awdp->delayed_wakeup.job = NULL; @@ -6072,7 +5672,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) for (i = 0; i <= erts_no_schedulers; i++) awdp->delayed_wakeup.sched2jix[i] = -1; } -#endif awdp->debug.wait_completed.flags = 0; awdp->debug.wait_completed.callback = NULL; awdp->debug.wait_completed.arg = NULL; @@ -6087,11 +5686,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, Uint64 time_stamp) { esdp->timer_wheel = NULL; -#ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; esdp->free_process = NULL; -#endif esdp->x_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, ERTS_X_REGS_ALLOCATED * @@ -6099,7 +5696,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#ifdef ERTS_DIRTY_SCHEDULERS esdp->run_queue = runq; if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -6127,19 +5723,14 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->dirty_shadow_process = shadow_proc; if (shadow_proc) { erts_init_empty_process(shadow_proc); - erts_smp_atomic32_init_nob(&shadow_proc->state, + erts_atomic32_init_nob(&shadow_proc->state, (ERTS_PSFLG_ACTIVE | ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_PROXY)); shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; } -#else - runq->scheduler = esdp; - esdp->run_queue = runq; - esdp->no = (Uint) num; - esdp->type = ERTS_SCHED_NORMAL; -#endif + ssi->esdp = esdp; esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; @@ -6160,9 +5751,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); -#ifdef ERTS_SMP *daww_ptr += daww_sz; -#endif } esdp->reductions = 0; @@ -6172,27 +5761,21 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, } void -erts_init_scheduling(int no_schedulers, int no_schedulers_online -#ifdef ERTS_DIRTY_SCHEDULERS - , int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online, +erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_threads, + int no_dirty_cpu_schedulers, int no_dirty_cpu_schedulers_online, int no_dirty_io_schedulers -#endif ) { int ix, n, no_ssi, tot_rqs; char *daww_ptr; size_t daww_sz; size_t size_runqs; -#ifdef ERTS_SMP erts_aint32_t set_schdlr_sspnd_change_flags; -#endif init_misc_op_list_alloc(); init_proc_sys_task_queues_alloc(); -#ifdef ERTS_SMP set_wakeup_other_data(); -#endif #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT if (erts_sched_balance_util) @@ -6202,12 +5785,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ASSERT(no_schedulers_online <= no_schedulers); ASSERT(no_schedulers_online >= 1); ASSERT(no_schedulers >= 1); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(no_dirty_cpu_schedulers <= no_schedulers); ASSERT(no_dirty_cpu_schedulers >= 1); ASSERT(no_dirty_cpu_schedulers_online <= no_schedulers_online); ASSERT(no_dirty_cpu_schedulers_online >= 1); -#endif + ASSERT(erts_no_poll_threads == no_poll_threads); /* Create and initialize run queues */ @@ -6216,9 +5798,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online size_runqs = sizeof(ErtsAlignedRunQueue) * tot_rqs; erts_aligned_run_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, size_runqs); -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&no_empty_run_queues, 0); -#endif + erts_atomic32_init_nob(&no_empty_run_queues, 0); erts_no_run_queues = n; @@ -6232,16 +5812,16 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online * id if the esdp->no <-> ix+1 mapping change. */ - erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); - erts_smp_cnd_init(&rq->cnd); + erts_mtx_init(&rq->mtx, "run_queue", make_small(ix + 1), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); + erts_cnd_init(&rq->cnd); -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP - if (ERTS_RUNQ_IX_IS_DIRTY(ix)) - erts_smp_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list"); + if (ERTS_RUNQ_IX_IS_DIRTY(ix)) { + erts_spinlock_init(&rq->sleepers.lock, "dirty_run_queue_sleep_list", + make_small(ix + 1), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); + } rq->sleepers.list = NULL; -#endif -#endif rq->waiting = 0; rq->woken = 0; @@ -6254,7 +5834,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online } rq->out_of_work_count = 0; rq->max_len = 0; - erts_smp_atomic32_set_nob(&rq->len, 0); + erts_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; @@ -6263,7 +5843,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online rq->procs.reductions = 0; for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - erts_smp_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0); + erts_atomic32_init_nob(&rq->procs.prio_info[pix].len, 0); rq->procs.prio_info[pix].max_len = 0; rq->procs.prio_info[pix].reds = 0; if (pix < ERTS_NO_PROC_PRIO_LEVELS - 1) { @@ -6275,7 +5855,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online rq->misc.start = NULL; rq->misc.end = NULL; - erts_smp_atomic32_init_nob(&rq->ports.info.len, 0); + erts_atomic32_init_nob(&rq->ports.info.len, 0); rq->ports.info.max_len = 0; rq->ports.info.reds = 0; rq->ports.start = NULL; @@ -6287,7 +5867,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online } -#ifdef ERTS_SMP if (erts_no_run_queues != 1) { run_queue_info = erts_alloc(ERTS_ALC_T_RUNQ_BLNS, @@ -6298,52 +5877,42 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online * erts_no_run_queues)); } -#endif n = (int) no_schedulers; erts_no_schedulers = n; erts_no_total_schedulers = n; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; erts_no_total_schedulers += no_dirty_cpu_schedulers; erts_no_dirty_io_schedulers = no_dirty_io_schedulers; erts_no_total_schedulers += no_dirty_io_schedulers; -#endif /* Create and initialize scheduler sleep info */ -#ifdef ERTS_SMP - no_ssi = n+1; -#else - no_ssi = 1; -#endif + no_ssi = n + 1 /* aux thread */; aligned_sched_sleep_info = erts_alloc_permanent_cache_aligned( ERTS_ALC_T_SCHDLR_SLP_INFO, no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo)); for (ix = 0; ix < no_ssi; ix++) { ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi; -#ifdef ERTS_SMP #if 0 /* no need to initialize these... */ ssi->next = NULL; ssi->prev = NULL; #endif - erts_smp_atomic32_init_nob(&ssi->flags, 0); + ssi->esdp = NULL; + erts_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ -#endif erts_atomic32_init_nob(&ssi->aux_work, 0); } -#ifdef ERTS_SMP - aligned_sched_sleep_info++; + aligned_sched_sleep_info += 1 /* aux thread */; -#ifdef ERTS_DIRTY_SCHEDULERS aligned_dirty_cpu_sched_sleep_info = erts_alloc_permanent_cache_aligned( ERTS_ALC_T_SCHDLR_SLP_INFO, no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo)); for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) { ErtsSchedulerSleepInfo *ssi = &aligned_dirty_cpu_sched_sleep_info[ix].ssi; - erts_smp_atomic32_init_nob(&ssi->flags, 0); + erts_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_dirty_cpu_thread_func */ erts_atomic32_init_nob(&ssi->aux_work, 0); } @@ -6353,24 +5922,29 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerSleepInfo)); for (ix = 0; ix < no_dirty_io_schedulers; ix++) { ErtsSchedulerSleepInfo *ssi = &aligned_dirty_io_sched_sleep_info[ix].ssi; - erts_smp_atomic32_init_nob(&ssi->flags, 0); + erts_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_dirty_io_thread_func */ erts_atomic32_init_nob(&ssi->aux_work, 0); } -#endif -#endif + + aligned_poll_thread_sleep_info = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_poll_threads*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_poll_threads; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_poll_thread_sleep_info[ix].ssi; + ssi->esdp = NULL; + erts_atomic32_init_nob(&ssi->flags, 0); + ssi->event = NULL; /* initialized in poll_thread */ + erts_atomic32_init_nob(&ssi->aux_work, 0); + } /* Create and initialize scheduler specific data */ -#ifdef ERTS_SMP daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob) + sizeof(int))*(n+1)); daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, daww_sz*n); -#else - daww_sz = 0; - daww_ptr = NULL; -#endif erts_aligned_scheduler_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, @@ -6383,7 +5957,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online NULL, 0); } -#ifdef ERTS_DIRTY_SCHEDULERS { Uint64 ts = sched_wall_time_ts(); int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers; @@ -6414,7 +5987,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online &adsp[adspix++].dsp, ts); } } -#endif init_misc_aux_work(); init_swtreq_alloc(); @@ -6423,19 +5995,22 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ debug_wait_completed_flags = 0; -#ifdef ERTS_SMP - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); + poll_thread_aux_work_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + no_poll_threads * sizeof(ErtsAuxWorkData)); + init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; - erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); + erts_mtx_init(&balance_info.update_mtx, "migration_info_update", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0); + erts_atomic32_init_nob(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -6466,7 +6041,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online suspend_run_queue(ERTS_RUNQ_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU, @@ -6483,7 +6057,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); - erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); } } @@ -6497,54 +6071,26 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ERTS_SCHED_DIRTY_IO, no_dirty_io_schedulers); - erts_smp_atomic32_init_nob(&dirty_count.cpu.active, + erts_atomic32_init_nob(&dirty_count.cpu.active, (erts_aint32_t) no_dirty_cpu_schedulers); - erts_smp_atomic32_init_nob(&dirty_count.io.active, + erts_atomic32_init_nob(&dirty_count.io.active, (erts_aint32_t) no_dirty_io_schedulers); -#endif if (set_schdlr_sspnd_change_flags) - erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, + erts_atomic32_set_nob(&schdlr_sspnd.changing, set_schdlr_sspnd_change_flags); - erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); - init_misc_aux_work(); -#else /* !ERTS_SMP */ - { - ErtsSchedulerData *esdp; - esdp = ERTS_SCHEDULER_IX(0); - erts_scheduler_data = esdp; -#ifdef USE_THREADS - erts_tsd_set(sched_data_key, (void *) esdp); -#endif - } - erts_no_dirty_cpu_schedulers = 0; - erts_no_dirty_io_schedulers = 0; -#endif - - erts_smp_atomic32_init_nob(&function_calls, 0); /* init port tasks */ erts_port_task_init(); -#ifndef ERTS_SMP -#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC - erts_scheduler_data->verify_unused_temp_alloc - = erts_alloc_get_verify_unused_temp_alloc( - &erts_scheduler_data->verify_unused_temp_alloc_data); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); -#endif -#endif - erts_smp_atomic32_init_relb(&erts_halt_progress, -1); + erts_atomic32_init_relb(&erts_halt_progress, -1); erts_halt_code = 0; -#if !defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - erts_lc_set_thread_name("scheduler 1"); -#endif } @@ -6557,7 +6103,6 @@ erts_schedid2runq(Uint id) return ERTS_RUNQ_IX(ix); } -#ifdef USE_THREADS ErtsSchedulerData * erts_get_scheduler_data(void) @@ -6565,16 +6110,13 @@ erts_get_scheduler_data(void) return (ErtsSchedulerData *) erts_tsd_get(sched_data_key); } -#endif static Process * make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) { erts_aint32_t state; Process *proxy; -#ifdef ERTS_SMP ErtsRunQueue *rq = RUNQ_READ_RQ(&proc->run_queue); -#endif state = (ERTS_PSFLG_PROXY | ERTS_PSFLG_IN_RUNQ @@ -6585,11 +6127,9 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) if (prev_proxy) { proxy = prev_proxy; - ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); - erts_smp_atomic32_set_nob(&proxy->state, state); -#ifdef ERTS_SMP + ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_atomic32_set_nob(&proxy->state, state); RUNQ_SET_RQ(&proc->run_queue, rq); -#endif } else { proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process)); @@ -6601,11 +6141,9 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) ui32[i] = (Uint32) 0xdeadbeef; } #endif - erts_smp_atomic32_init_nob(&proxy->state, state); -#ifdef ERTS_SMP - erts_smp_atomic_init_nob(&proxy->run_queue, - erts_smp_atomic_read_nob(&proc->run_queue)); -#endif + erts_atomic32_init_nob(&proxy->state, state); + erts_atomic_init_nob(&proxy->run_queue, + erts_atomic_read_nob(&proc->run_queue)); } proxy->common.id = proc->common.id; @@ -6618,7 +6156,6 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 -#ifdef ERTS_DIRTY_SCHEDULERS static int check_dirty_enqueue_in_prio_queue(Process *c_p, @@ -6649,7 +6186,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, if ((*newp) & ERTS_PSFLG_ACTIVE_SYS) return ERTS_ENQUEUE_NORMAL_QUEUE; - dact = erts_smp_atomic32_read_mb(&c_p->dirty_state); + dact = erts_atomic32_read_mb(&c_p->dirty_state); if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS | ERTS_PSFLG_DIRTY_CPU_PROC)) { max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET) @@ -6694,7 +6231,7 @@ fin_dirty_enq_s_change(Process *p, erts_aint32_t qbit = 1 << enq_prio; qbit <<= qmask_offset; - if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) { + if (qbit & erts_atomic32_read_bor_mb(&p->dirty_state, qbit)) { /* Already enqueue by someone else... */ if (pstruct_reserved) { /* We reserved process struct for enqueue; clear it... */ @@ -6703,7 +6240,7 @@ fin_dirty_enq_s_change(Process *p, #else (void) #endif - erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); + erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); ASSERT(old & ERTS_PSFLG_IN_RUNQ); } return 0; @@ -6712,7 +6249,6 @@ fin_dirty_enq_s_change(Process *p, return !0; } -#endif /* ERTS_DIRTY_SCHEDULERS */ static ERTS_INLINE int check_enqueue_in_prio_queue(Process *c_p, @@ -6727,14 +6263,12 @@ check_enqueue_in_prio_queue(Process *c_p, *prq_prio_p = aprio; -#ifdef ERTS_DIRTY_SCHEDULERS if (actual & ERTS_PSFLGS_DIRTY_WORK) { int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual, aprio, qbit); if (res != ERTS_ENQUEUE_NORMAL_QUEUE) return res; } -#endif max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; @@ -6777,7 +6311,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st return NULL; -#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: @@ -6798,7 +6331,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st return NULL; -#endif default: { ErtsRunQueue* runq; @@ -6808,7 +6340,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st runq = erts_get_runq_proc(p); -#ifdef ERTS_SMP if (!(ERTS_PSFLG_BOUND & state)) { ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); if (new_runq) { @@ -6816,7 +6347,6 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st runq = new_runq; } } -#endif ASSERT(runq); @@ -6844,7 +6374,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; else { running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) { /* @@ -6858,11 +6387,10 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, */ ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE))); - state = erts_smp_atomic32_read_band_nob(&p->state, + state = erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; } -#endif } a = state; @@ -6879,7 +6407,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } - a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } @@ -6891,7 +6419,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, if (erts_system_profile_flags.runnable_procs) { /* Status lock prevents out of order "runnable proc" trace msgs */ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { @@ -6903,15 +6431,10 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, if (proxy) free_proxy_proc(proxy); - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); -#if !defined(ERTS_SMP) - /* Decrement refc if process struct is free... */ - return !!(n & ERTS_PSFLG_FREE); -#else /* Decrement refc if scheduled out from dirty scheduler... */ return !is_normal_sched; -#endif } else { Process* sched_p; @@ -6930,7 +6453,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, ASSERT(runq); - erts_smp_runq_lock(runq); + erts_runq_lock(runq); if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */ @@ -6941,11 +6464,11 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, if (runq == c_rq) return 0; - erts_smp_runq_unlock(runq); + erts_runq_unlock(runq); smp_notify_inc_runq(runq); - erts_smp_runq_lock(c_rq); + erts_runq_lock(c_rq); /* * Decrement refc if process is scheduled out by a @@ -6993,12 +6516,12 @@ add2runq(int enqueue, erts_aint32_t prio, sched_p = make_proxy_proc(pxy, proc, prio); } - erts_smp_runq_lock(runq); + erts_runq_lock(runq); /* Enqueue the process */ enqueue_process(runq, (int) prio, sched_p); - erts_smp_runq_unlock(runq); + erts_runq_unlock(runq); smp_notify_inc_runq(runq); } } @@ -7023,7 +6546,7 @@ change_proc_schedule_state(Process *p, unsigned int lock_status = (prof_runnable_procs && !(locks & ERTS_PROC_LOCK_STATUS)); - ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p)); ASSERT(!(a & ERTS_PSFLG_PROXY)); ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING @@ -7038,7 +6561,7 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_ACTIVE_SYS)) == 0); if (lock_status) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); while (1) { erts_aint32_t e; @@ -7062,11 +6585,9 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_IN_RUNQ | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE -#ifdef ERTS_DIRTY_SCHEDULERS || (n & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING -#endif ) { /* * Active and seemingly need to be enqueued, but @@ -7076,7 +6597,7 @@ change_proc_schedule_state(Process *p, enqueue = check_enqueue_in_prio_queue(p, enq_prio_p, &n, a); } - a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; if (enqueue == ERTS_ENQUEUE_NOT && n == a) @@ -7101,7 +6622,7 @@ change_proc_schedule_state(Process *p, } if (lock_status) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } @@ -7145,7 +6666,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); prof_runnable_procs = erts_system_profile_flags.runnable_procs; locked = 0; free_stqs = NULL; @@ -7165,9 +6686,9 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, if (!locked) { locked = 1; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); if (state & fail_state) { *fail_state_p = (state & fail_state); free_stqs = stqs; @@ -7211,7 +6732,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, n = e = a; n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); - a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e); + a = erts_atomic32_cmpxchg_nob(&p->state, n, e); } while (a != e); state = n; } @@ -7221,10 +6742,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, enq_prio = -1; /* Status lock prevents out of order "runnable proc" trace msgs */ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); if (!prof_runnable_procs) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -7244,7 +6765,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, | ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS))) enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); - a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; if (a == n && enqueue == ERTS_ENQUEUE_NOT) @@ -7263,7 +6784,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, profile_runnable_proc(p, am_active); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -7272,12 +6793,12 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, cleanup: if (locked) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); if (free_stqs) proc_sys_task_queues_free(free_stqs); - ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); return res; } @@ -7287,15 +6808,15 @@ suspend_process(Process *c_p, Process *p) { erts_aint32_t state; int suspended = 0; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); if ((state & ERTS_PSFLG_SUSPENDED)) suspended = -1; else { if (c_p == p) { - state = erts_smp_atomic32_read_bor_relb(&p->state, + state = erts_atomic32_read_bor_relb(&p->state, ERTS_PSFLG_SUSPENDED); ASSERT(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS @@ -7311,7 +6832,7 @@ suspend_process(Process *c_p, Process *p) n = e = state; n |= ERTS_PSFLG_SUSPENDED; - state = erts_smp_atomic32_cmpxchg_relb(&p->state, n, e); + state = erts_atomic32_cmpxchg_relb(&p->state, n, e); if (state == e) { suspended = 1; break; @@ -7356,14 +6877,14 @@ resume_process(Process *p, ErtsProcLocks locks) erts_aint32_t state, enq_prio = -1; int enqueue; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); ASSERT(p->rcount > 0); if (--p->rcount > 0) /* multiple suspend */ return; - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); enqueue = change_proc_schedule_state(p, ERTS_PSFLG_SUSPENDED, 0, @@ -7373,7 +6894,6 @@ resume_process(Process *p, ErtsProcLocks locks) add2runq(enqueue, enq_prio, p, state, NULL); } -#ifdef ERTS_SMP static ERTS_INLINE void sched_resume_wake__(ErtsSchedulerSleepInfo *ssi) @@ -7384,7 +6904,7 @@ sched_resume_wake__(ErtsSchedulerSleepInfo *ssi) | ERTS_SSI_FLG_SUSPENDED); erts_aint32_t oflgs; do { - oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs); + oflgs = erts_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs); if (oflgs == xflgs) { erts_sched_finish_poke(ssi, oflgs); break; @@ -7399,7 +6919,6 @@ nrml_sched_ix_resume_wake(Uint ix) sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS static void dcpu_sched_ix_resume_wake(Uint ix) @@ -7413,7 +6932,6 @@ dio_sched_ix_resume_wake(Uint ix) sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); } -#endif static erts_aint32_t sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) @@ -7425,7 +6943,7 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) erts_aint32_t xflgs = xpct; do { - oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -7442,7 +6960,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) erts_aint32_t flgs; do { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + flgs = erts_atomic32_read_acqb(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) @@ -7461,21 +6979,23 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) } static erts_aint32_t -sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t sleep_type) { erts_aint32_t oflgs; - erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t nflgs = ((ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED) + | sleep_type); erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED); + ASSERT(sleep_type == ERTS_SSI_FLG_TSE_SLEEPING); erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + oflgs = erts_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING @@ -7493,11 +7013,11 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) static void init_scheduler_suspend(void) { - erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); + erts_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); schdlr_sspnd.online.normal = 1; schdlr_sspnd.curr_online.normal = 1; schdlr_sspnd.active.normal = 1; -#ifdef ERTS_DIRTY_SCHEDULERS schdlr_sspnd.online.dirty_cpu = 0; schdlr_sspnd.curr_online.dirty_cpu = 0; schdlr_sspnd.active.dirty_cpu = 0; @@ -7505,8 +7025,7 @@ init_scheduler_suspend(void) schdlr_sspnd.curr_online.dirty_io = 0; schdlr_sspnd.active.dirty_io = 0; schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO; -#endif - erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); + erts_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.chngq = NULL; schdlr_sspnd.changer = am_false; schdlr_sspnd.nmsb.ongoing = 0; @@ -7537,7 +7056,7 @@ schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid) : 0)); if (p) { resume_process(p, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); if (sched_type != ERTS_SCHED_NORMAL) erts_proc_dec_refc(p); } @@ -7566,7 +7085,6 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, } } -#ifdef ERTS_DIRTY_SCHEDULERS static ERTS_INLINE int have_dirty_work(void) @@ -7726,17 +7244,6 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, if (exec_type != ERTS_SCHED_NORMAL) schdlr_sspnd.last_msb_dirty_type = exec_type; else { - erts_aint32_t calls; - /* - * Going back to normal scheduler after - * dirty execution; make sure it will check - * for I/O... - */ - if (ERTS_USE_MODIFIED_TIMING()) - calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1; - else - calls = INPUT_REDUCTIONS + 1; - erts_smp_atomic32_set_nob(&function_calls, calls); if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT) & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT) @@ -7771,7 +7278,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, #else (void) #endif - erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags, + erts_atomic32_read_bset_mb(&esdp->ssi->flags, (ERTS_SSI_FLG_SUSPENDED | ERTS_SSI_FLG_MSB_EXEC), ERTS_SSI_FLG_SUSPENDED); @@ -7798,7 +7305,7 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, #else (void) #endif - erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, + erts_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, (ERTS_SSI_FLG_SUSPENDED | ERTS_SSI_FLG_MSB_EXEC), ERTS_SSI_FLG_MSB_EXEC); @@ -7810,7 +7317,34 @@ msb_scheduler_type_switch(ErtsSchedType sched_type, } -#endif +static ERTS_INLINE void +suspend_normal_scheduler_sleep(ErtsSchedulerData *esdp) +{ + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + erts_aint32_t flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } +} + +static ERTS_INLINE void +suspend_dirty_scheduler_sleep(ErtsSchedulerData *esdp) +{ + suspend_normal_scheduler_sleep(esdp); +} static void suspend_scheduler(ErtsSchedulerData *esdp) @@ -7838,14 +7372,6 @@ suspend_scheduler(ErtsSchedulerData *esdp) */ -#if !defined(ERTS_DIRTY_SCHEDULERS) - - sched_type = ERTS_SCHED_NORMAL; - online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; - no = esdp->no; - ASSERT(no != 1); - -#else sched_type = esdp->type; switch (sched_type) { @@ -7866,25 +7392,24 @@ suspend_scheduler(ErtsSchedulerData *esdp) return; } - if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) { + if (erts_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) { ASSERT(no == 1); if (!msb_scheduler_type_switch(sched_type, esdp, no)) return; /* Suspend and let scheduler 1 of another type execute... */ } -#endif if (sched_type != ERTS_SCHED_NORMAL) { dirty_active(esdp, -1); - erts_smp_runq_unlock(esdp->run_queue); + erts_runq_unlock(esdp->run_queue); dirty_sched_wall_time_change(esdp, 0); } else { if (no != 1) evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); + erts_runq_unlock(esdp->run_queue); erts_sched_check_cpu_bind_prep_suspend(esdp); @@ -7892,14 +7417,14 @@ suspend_scheduler(ErtsSchedulerData *esdp) profile_scheduler(make_small(esdp->no), am_inactive); } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); while (1) { @@ -7931,7 +7456,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (clr_flg) { ErtsProcList *plp, *end_plp; - changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + changing = erts_atomic32_read_band_nob(&schdlr_sspnd.changing, ~clr_flg); changing &= ~clr_flg; (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); @@ -7977,7 +7502,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) == schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, sched_type))) { ErtsProcList *plp; - changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + changing = erts_atomic32_read_band_nob(&schdlr_sspnd.changing, ~online_flag); changing &= ~online_flag; if (sched_type == ERTS_SCHED_NORMAL) { @@ -7999,23 +7524,24 @@ suspend_scheduler(ErtsSchedulerData *esdp) } if (curr_online) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + flgs = erts_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); schdlr_sspnd_resume_procs(sched_type, &resume); while (1) { - ErtsMonotonicTime current_time; - erts_aint32_t flgs; - if (sched_type != ERTS_SCHED_NORMAL) - aux_work = 0; - else { + suspend_dirty_scheduler_sleep(esdp); + else + { + ErtsMonotonicTime current_time, timeout_time; int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue); + ASSERT(sched_type == ERTS_SCHED_NORMAL); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|evacuate) { @@ -8031,123 +7557,62 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); if (evacuate) { - erts_smp_runq_lock(esdp->run_queue); + erts_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); + erts_runq_unlock(esdp->run_queue); } } - } - if (aux_work) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - erts_bump_timers(esdp->timer_wheel, current_time); - } - } - else { - ErtsMonotonicTime timeout_time; - int do_timeout; + if (aux_work) + timeout_time = erts_next_timeout_time(esdp->next_tmo_ref); + else + timeout_time = erts_check_next_timeout_time(esdp); - if (sched_type == ERTS_SCHED_NORMAL) { - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - do_timeout = (current_time >= timeout_time); - } - else { - timeout_time = ERTS_MONOTONIC_TIME_MAX; - current_time = 0; - do_timeout = 0; - } + current_time = erts_get_monotonic_time(esdp); - if (do_timeout) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - else { - if (sched_type == ERTS_SCHED_NORMAL) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); - } - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - if (sched_type == ERTS_SCHED_NORMAL) - current_time = erts_get_monotonic_time(esdp); - else - current_time = 0; - - do { - Sint64 timeout; - if (current_time >= timeout_time) - break; - if (sched_type != ERTS_SCHED_NORMAL) - timeout = -1; - else - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; - res = erts_tse_twait(ssi->event, timeout); - - if (sched_type == ERTS_SCHED_NORMAL) - current_time = erts_get_monotonic_time(esdp); - else - current_time = 0; - - } while (res == EINTR); - } - } - if (sched_type == ERTS_SCHED_NORMAL) - erts_thr_progress_finalize_wait(esdp); - } + if (!aux_work && current_time < timeout_time) { + /* go to sleep... */ + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(NULL); + suspend_normal_scheduler_sleep(esdp); + erts_thr_progress_finalize_wait(NULL); + current_time = erts_get_monotonic_time(esdp); + } - if (current_time >= timeout_time) { - ASSERT(sched_type == ERTS_SCHED_NORMAL); - erts_bump_timers(esdp->timer_wheel, current_time); - } - } + if (current_time >= timeout_time) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) break; } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + erts_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); } schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); if (changing) { if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && !schdlr_sspnd.msb.ongoing && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online, &schdlr_sspnd.active)) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + erts_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) @@ -8156,14 +7621,14 @@ suspend_scheduler(ErtsSchedulerData *esdp) ERTS_SCHED_NORMAL) == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL))) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + erts_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_NMSB); } } ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); schdlr_sspnd_resume_procs(sched_type, &resume); @@ -8182,7 +7647,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - erts_smp_runq_lock(esdp->run_queue); + erts_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); if (sched_type != ERTS_SCHED_NORMAL) @@ -8206,7 +7671,7 @@ erts_schedulers_state(Uint *total, { if (active || online || dirty_cpu_online || dirty_cpu_active || dirty_io_active) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); if (active) *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL); @@ -8222,7 +7687,7 @@ erts_schedulers_state(Uint *total, if (dirty_io_active) *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_DIRTY_IO); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); } if (total) @@ -8238,7 +7703,7 @@ abort_sched_onln_chng_waitq(Process *p) { Eterm resume = NIL; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); #ifdef DEBUG { @@ -8288,7 +7753,7 @@ abort_sched_onln_chng_waitq(Process *p) } } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); if (is_internal_pid(resume)) schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume); @@ -8305,11 +7770,7 @@ erts_set_schedulers_online(Process *p, erts_aint32_t changing = 0, change_flags; int online, increase; ErtsProcList *plp; -#ifdef ERTS_DIRTY_SCHEDULERS int dirty_no, change_dirty, dirty_online; -#else - ASSERT(!dirty_only); -#endif if (new_no < 1) return ERTS_SCHDLR_SSPND_EINVAL; @@ -8318,11 +7779,9 @@ erts_set_schedulers_online(Process *p, else if (erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; -#ifdef ERTS_DIRTY_SCHEDULERS if (dirty_only) resume_proc = 0; else -#endif { resume_proc = 1; /* @@ -8331,23 +7790,21 @@ erts_set_schedulers_online(Process *p, * race... */ if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); suspend_process(p, p); if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); change_flags = 0; have_unlocked_plocks = 0; no = (int) new_no; -#ifdef ERTS_DIRTY_SCHEDULERS if (!dirty_only) -#endif { - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + changing = erts_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { enqueue_wait: p->flags |= F_SCHDLR_ONLN_WAITQ; @@ -8373,12 +7830,6 @@ erts_set_schedulers_online(Process *p, *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_NORMAL); -#ifndef ERTS_DIRTY_SCHEDULERS - if (no == online) { - res = ERTS_SCHDLR_SSPND_DONE; - goto done; - } -#else /* ERTS_DIRTY_SCHEDULERS */ dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU); if (dirty_only) @@ -8430,7 +7881,6 @@ erts_set_schedulers_online(Process *p, if (dirty_only) increase = (dirty_no > dirty_online); else -#endif /* ERTS_DIRTY_SCHEDULERS */ { change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, @@ -8439,12 +7889,11 @@ erts_set_schedulers_online(Process *p, increase = (no > online); } - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags); + erts_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags); res = ERTS_SCHDLR_SSPND_DONE; if (increase) { int ix; -#ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { ErtsSchedulerSleepInfo* ssi; if (schdlr_sspnd.msb.ongoing) { @@ -8458,7 +7907,6 @@ erts_set_schedulers_online(Process *p, } } if (!dirty_only) -#endif { if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { for (ix = online; ix < no; ix++) @@ -8467,7 +7915,7 @@ erts_set_schedulers_online(Process *p, else { if (plocks) { have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); } change_no_used_runqs(no); @@ -8480,7 +7928,6 @@ erts_set_schedulers_online(Process *p, } } else /* if decrease */ { -#ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { if (schdlr_sspnd.msb.ongoing) { for (ix = dirty_no; ix < dirty_online; ix++) @@ -8498,7 +7945,6 @@ erts_set_schedulers_online(Process *p, } } if (!dirty_only) -#endif { if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { for (ix = no; ix < online; ix++) @@ -8507,7 +7953,7 @@ erts_set_schedulers_online(Process *p, else { if (plocks) { have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); } change_no_used_runqs(no); @@ -8536,17 +7982,17 @@ done: <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_NORMAL)); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); if (have_unlocked_plocks) - erts_smp_proc_lock(p, plocks); + erts_proc_lock(p, plocks); if (resume_proc) { if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } return res; @@ -8584,13 +8030,13 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal else { resume_proc = 1; if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); suspend_process(p, p); if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); if (on) { /* ------ BLOCK ------ */ if (msbp->chngq) { ASSERT(msbp->ongoing); @@ -8620,12 +8066,12 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal p->flags |= have_blckd_flg; if (plocks) { have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); } ASSERT(!msbp->ongoing); msbp->ongoing = 1; - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + erts_atomic32_read_bor_nob(&schdlr_sspnd.changing, chng_flg); change_no_used_runqs(1); for (ix = 1; ix < erts_no_run_queues; ix++) @@ -8636,21 +8082,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal wake_scheduler(rq); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal) { ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC); - erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, + erts_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, ERTS_SSI_FLG_MSB_EXEC); for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) dcpu_sched_ix_suspend_wake(ix); for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) dio_sched_ix_suspend_wake(ix); } -#endif wait_until_msb: - ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); + ASSERT(chng_flg & erts_atomic32_read_nob(&schdlr_sspnd.changing)); plp = proclist_create(p); erts_proclist_store_last(&msbp->chngq, plp); @@ -8691,14 +8135,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } if (!msbp->blckrs && !msbp->chngq) { int online; - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + erts_atomic32_read_bor_nob(&schdlr_sspnd.changing, chng_flg); p->flags &= ~have_blckd_flg; msbp->ongoing = 0; if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { if (plocks) { have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); + erts_proc_unlock(p, plocks); } online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, @@ -8712,7 +8156,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal for (ix = online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!schdlr_sspnd.msb.ongoing) { /* Get rid of msb-exec flag in run-queue of scheduler 1 */ resume_run_queue(ERTS_RUNQ_IX(0)); @@ -8723,7 +8166,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) dio_sched_ix_resume_wake(ix); } -#endif } unblock_res: @@ -8735,17 +8177,17 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal res = ERTS_SCHDLR_SSPND_DONE; } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); if (have_unlocked_plocks) - erts_smp_proc_lock(p, plocks); + erts_proc_lock(p, plocks); if (resume_proc) { if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); if (!(plocks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } return res; @@ -8755,14 +8197,14 @@ int erts_is_multi_scheduling_blocked(void) { int res; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); if (schdlr_sspnd.msb.blckrs) res = 1; else if (schdlr_sspnd.nmsb.blckrs) res = -1; else res = 0; - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); return res; } @@ -8774,7 +8216,7 @@ erts_multi_scheduling_blockers(Process *p, int normal) msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_mtx_lock(&schdlr_sspnd.mtx); if (!erts_proclist_is_empty(msbp->blckrs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; @@ -8802,7 +8244,7 @@ erts_multi_scheduling_blockers(Process *p, int normal) } HRelease(p, hp_end, hp); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_mtx_unlock(&schdlr_sspnd.mtx); return res; } @@ -8812,10 +8254,9 @@ sched_thread_func(void *vesdp) ErtsThrPrgrCallbacks callbacks; ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; -#ifdef ERTS_SMP erts_tse_t *tse; -#endif + erts_port_task_pre_alloc_init_thread(); erts_sched_init_time_sup(esdp); if (no == 1) @@ -8824,7 +8265,6 @@ sched_thread_func(void *vesdp) (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, ERTS_RUNQ_FLG_EXEC); -#ifdef ERTS_SMP tse = erts_tse_fetch(); erts_tse_prepare_timed(tse); ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse; @@ -8838,7 +8278,6 @@ sched_thread_func(void *vesdp) erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); erts_alloc_register_scheduler(vesdp); -#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; @@ -8847,18 +8286,14 @@ sched_thread_func(void *vesdp) } #endif erts_tsd_set(sched_data_key, vesdp); -#ifdef ERTS_SMP #if HAVE_ERTS_MSEG erts_mseg_late_init(); #endif -#if ERTS_USE_ASYNC_READY_Q esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no); -#endif erts_sched_init_check_cpu_bind(esdp); erts_proc_lock_prepare_proc_lock_waiter(); -#endif #ifdef HIPE hipe_thread_signal_init(); @@ -8883,8 +8318,6 @@ sched_thread_func(void *vesdp) return NULL; } -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP static void* sched_dirty_cpu_thread_func(void *vesdp) { @@ -8914,9 +8347,7 @@ sched_dirty_cpu_thread_func(void *vesdp) } #endif erts_tsd_set(sched_data_key, vesdp); -#if ERTS_USE_ASYNC_READY_Q esdp->aux_work_data.async_ready.queue = NULL; -#endif erts_proc_lock_prepare_proc_lock_waiter(); @@ -8962,9 +8393,7 @@ sched_dirty_io_thread_func(void *vesdp) } #endif erts_tsd_set(sched_data_key, vesdp); -#if ERTS_USE_ASYNC_READY_Q esdp->aux_work_data.async_ready.queue = NULL; -#endif erts_proc_lock_prepare_proc_lock_waiter(); @@ -8980,40 +8409,38 @@ sched_dirty_io_thread_func(void *vesdp) no); return NULL; } -#endif -#endif - -static ethr_tid aux_tid; void erts_start_schedulers(void) { + ethr_tid tid; int res = 0; Uint actual; Uint wanted = erts_no_schedulers; Uint wanted_no_schedulers = erts_no_schedulers; char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; + int ix; opts.detached = 1; opts.name = name; -#ifdef ERTS_SMP if (erts_runq_supervision_interval) { opts.suggested_stack_size = 16; erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); - if (0 != ethr_thr_create(&runq_supervisor_tid, - runq_supervisor, - NULL, - &opts)) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); + res = ethr_thr_create(&runq_supervisor_tid, + runq_supervisor, + NULL, + &opts); + if (0 != res) + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread, " + "error = %d\n", res); } -#endif opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; @@ -9039,17 +8466,14 @@ erts_start_schedulers(void) } erts_no_schedulers = actual; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP { - int ix; for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); @@ -9057,19 +8481,25 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); } } -#endif -#endif ERTS_THR_MEMORY_BARRIER; erts_snprintf(opts.name, 16, "aux"); - res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + res = ethr_thr_create(&tid, aux_thread, NULL, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread, error = %d\n", res); + + for (ix = 0; ix < erts_no_poll_threads; ix++) { + erts_snprintf(opts.name, 16, "%d_poller", ix); + + res = ethr_thr_create(&tid, poll_thread, (void*)(UWord)ix, &opts); + if (res != 0) + erts_exit(ERTS_ERROR_EXIT, "Failed to create poll thread\n"); + } if (actual < 1) erts_exit(ERTS_ERROR_EXIT, @@ -9088,9 +8518,7 @@ erts_start_schedulers(void) } } -#endif /* ERTS_SMP */ -#ifdef ERTS_SMP static void add_pend_suspend(Process *suspendee, @@ -9126,7 +8554,7 @@ handle_pending_suspend(Process *p, ErtsProcLocks p_locks) ErtsPendingSuspend *psp; int is_alive = !ERTS_PROC_IS_EXITING(p); - ERTS_SMP_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS); + ERTS_LC_ASSERT(p_locks & ERTS_PROC_LOCK_STATUS); /* * New pending suspenders might appear while we are processing @@ -9152,15 +8580,15 @@ cancel_suspend_of_suspendee(Process *p, ErtsProcLocks p_locks) if (is_not_nil(p->suspendee)) { Process *rp; if (!(p_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); rp = erts_pid2proc(p, p_locks|ERTS_PROC_LOCK_STATUS, p->suspendee, ERTS_PROC_LOCK_STATUS); if (rp) { erts_resume(rp, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); } if (!(p_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); p->suspendee = NIL; } } @@ -9173,7 +8601,7 @@ handle_pend_sync_suspend(Process *suspendee, { Process *suspender; - ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); suspender = erts_pid2proc(suspendee, suspendee_locks, @@ -9189,7 +8617,7 @@ handle_pend_sync_suspend(Process *suspendee, resume suspender */ ASSERT(suspendee != suspender); resume_process(suspender, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -9200,10 +8628,10 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, Process *rp; int unlock_c_p_status; - ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); - ERTS_SMP_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)); + ERTS_LC_ASSERT(c_p_locks & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(pid_locks & (ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS)); if (c_p->common.id == pid) return erts_pid2proc(c_p, c_p_locks, pid, pid_locks); @@ -9212,7 +8640,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, unlock_c_p_status = 0; else { unlock_c_p_status = 1; - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); } if (c_p->suspendee == pid) { @@ -9251,21 +8679,19 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, /* Other process running */ ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING) - & erts_smp_atomic32_read_nob(&rp->state)); + & erts_atomic32_read_nob(&rp->state)); -#ifdef ERTS_DIRTY_SCHEDULERS if (!suspend - && (erts_smp_atomic32_read_nob(&rp->state) + && (erts_atomic32_read_nob(&rp->state) & ERTS_PSFLG_DIRTY_RUNNING)) { ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; - if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) { + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, pid, pid_locks|ERTS_PROC_LOCK_STATUS); } goto done; } -#endif running: @@ -9285,19 +8711,19 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, c_p->flags |= F_P2PNR_RESCHED; } /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); rp = ERTS_PROC_LOCK_BUSY; } else { ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; - if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (need_locks && erts_proc_trylock(rp, need_locks) == EBUSY) { if ((ERTS_PSFLG_RUNNING_SYS|ERTS_PSFLG_DIRTY_RUNNING_SYS) - & erts_smp_atomic32_read_nob(&rp->state)) { + & erts_atomic32_read_nob(&rp->state)) { /* Executing system task... */ resume_process(rp, ERTS_PROC_LOCK_STATUS); goto running; } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); /* * If we are unlucky, the process just got selected for * execution of a system task. In this case we may be @@ -9321,7 +8747,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, #ifdef DEBUG { erts_aint32_t state; - state = erts_smp_atomic32_read_nob(&rp->state); + state = erts_atomic32_read_nob(&rp->state); ASSERT((state & ERTS_PSFLG_PENDING_EXIT) || !(state & ERTS_PSFLG_RUNNING)); } @@ -9335,9 +8761,9 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, done: if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); return rp; } @@ -9383,7 +8809,7 @@ do_bif_suspend_process(Process *c_p, { ASSERT(suspendee); ASSERT(!ERTS_PROC_IS_EXITING(suspendee)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(suspendee)); if (smon) { if (!smon->active) { @@ -9406,7 +8832,7 @@ handle_pend_bif_sync_suspend(Process *suspendee, { Process *suspender; - ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); suspender = erts_pid2proc(suspendee, suspendee_locks, @@ -9435,7 +8861,7 @@ handle_pend_bif_sync_suspend(Process *suspendee, resume suspender */ ASSERT(suspender != suspendee); resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(suspender, + erts_proc_unlock(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); } } @@ -9449,7 +8875,7 @@ handle_pend_bif_async_suspend(Process *suspendee, Process *suspender; - ERTS_SMP_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); + ERTS_LC_ASSERT(suspendee_locks & ERTS_PROC_LOCK_STATUS); suspender = erts_pid2proc(suspendee, suspendee_locks, @@ -9473,11 +8899,10 @@ handle_pend_bif_async_suspend(Process *suspendee, do_bif_suspend_process(suspendee, smon, suspendee); ASSERT(!smon || res != 0); } - erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(suspender, ERTS_PROC_LOCK_LINK); } } -#endif /* ERTS_SMP */ /* * The erlang:suspend_process/2 BIF @@ -9526,7 +8951,7 @@ suspend_process_2(BIF_ALIST_2) ? (ErtsProcLocks) 0 : ERTS_PROC_LOCK_STATUS); - erts_smp_proc_lock(BIF_P, xlocks); + erts_proc_lock(BIF_P, xlocks); suspendee = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN|xlocks, @@ -9537,34 +8962,15 @@ suspend_process_2(BIF_ALIST_2) smon = erts_add_or_lookup_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); -#ifndef ERTS_SMP /* no ERTS_SMP */ - - /* This is really a piece of cake without SMP support... */ - if (!smon->active) { - erts_smp_atomic32_read_bor_nob(&suspendee->state, ERTS_PSFLG_SUSPENDED); - suspend_process(BIF_P, suspendee); - smon->active++; - res = am_true; - } - else if (unless_suspending) - res = am_false; - else if (smon->active == INT_MAX) - goto system_limit; - else { - smon->active++; - res = am_true; - } - -#else /* ERTS_SMP */ /* ... but a little trickier with SMP support ... */ if (asynchronous) { /* --- Asynchronous suspend begin ---------------------------------- */ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_LINK + ERTS_LC_ASSERT(ERTS_PROC_LOCK_LINK & erts_proc_lc_my_proc_locks(BIF_P)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS == erts_proc_lc_my_proc_locks(suspendee)); if (smon->active) { @@ -9604,10 +9010,10 @@ suspend_process_2(BIF_ALIST_2) else /* if (!asynchronous) */ { /* --- Synchronous suspend begin ----------------------------------- */ - ERTS_SMP_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS) + ERTS_LC_ASSERT(((ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS) & erts_proc_lc_my_proc_locks(BIF_P)) == (ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS == erts_proc_lc_my_proc_locks(suspendee)); if (BIF_P->suspendee == BIF_ARG_1) { @@ -9673,10 +9079,9 @@ suspend_process_2(BIF_ALIST_2) /* --- Synchronous suspend end ------------------------------------- */ } -#endif /* ERTS_SMP */ #ifdef DEBUG { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&suspendee->state); + erts_aint32_t state = erts_atomic32_read_acqb(&suspendee->state); ASSERT((state & ERTS_PSFLG_SUSPENDED) || (asynchronous && smon->pending)); ASSERT((state & ERTS_PSFLG_SUSPENDED) @@ -9684,8 +9089,8 @@ suspend_process_2(BIF_ALIST_2) } #endif - erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(BIF_P, xlocks); + erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(BIF_P, xlocks); BIF_RET(res); system_limit: @@ -9693,26 +9098,22 @@ suspend_process_2(BIF_ALIST_2) goto do_return; no_suspendee: -#ifdef ERTS_SMP BIF_P->suspendee = NIL; -#endif erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); badarg: ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); -#ifdef ERTS_SMP goto do_return; yield: ERTS_BIF_PREP_YIELD2(res, bif_export[BIF_suspend_process_2], BIF_P, BIF_ARG_1, BIF_ARG_2); -#endif do_return: if (suspendee) - erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); if (xlocks) - erts_smp_proc_unlock(BIF_P, xlocks); + erts_proc_unlock(BIF_P, xlocks); return res; } @@ -9732,7 +9133,7 @@ resume_process_1(BIF_ALIST_1) if (BIF_P->common.id == BIF_ARG_1) BIF_ERROR(BIF_P, BADARG); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK); smon = erts_lookup_suspend_monitor(BIF_P->suspend_monitors, BIF_ARG_1); if (!smon) { @@ -9778,17 +9179,17 @@ resume_process_1(BIF_ALIST_1) goto no_suspendee; ASSERT(ERTS_PSFLG_SUSPENDED - & erts_smp_atomic32_read_nob(&suspendee->state)); + & erts_atomic32_read_nob(&suspendee->state)); ASSERT(BIF_P != suspendee); resume_process(suspendee, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } if (!smon->active && !smon->pending) erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); BIF_RET(am_true); @@ -9797,7 +9198,7 @@ resume_process_1(BIF_ALIST_1) erts_delete_suspend_monitor(&BIF_P->suspend_monitors, BIF_ARG_1); error: - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK); BIF_ERROR(BIF_P, BADARG); } @@ -9806,18 +9207,16 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) { if (is_not_internal_pid(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); -#ifdef ERTS_DIRTY_SCHEDULERS else { Process *rp = erts_proc_lookup(BIF_ARG_1); if (rp) { - erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); if (state & (ERTS_PSFLG_DIRTY_RUNNING |ERTS_PSFLG_DIRTY_RUNNING_SYS)) { BIF_RET(am_true); } } } -#endif BIF_RET(am_false); } @@ -9827,28 +9226,26 @@ run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int inc Sint rq_len; if (locked) - rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + rq_len = (Sint) erts_atomic32_read_dirty(&rq->len); else - rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + rq_len = (Sint) erts_atomic32_read_nob(&rq->len); ASSERT(rq_len >= 0); if (incl_active_sched) { -#ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { erts_aint32_t dcnt; if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(rq)) { - dcnt = erts_smp_atomic32_read_nob(&dirty_count.cpu.active); + dcnt = erts_atomic32_read_nob(&dirty_count.cpu.active); ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_cpu_schedulers); } else { ASSERT(ERTS_RUNQ_IS_DIRTY_IO_RUNQ(rq)); - dcnt = erts_smp_atomic32_read_nob(&dirty_count.io.active); + dcnt = erts_atomic32_read_nob(&dirty_count.io.active); ASSERT(0 <= dcnt && dcnt <= erts_no_dirty_io_schedulers); } rq_len += (Sint) dcnt; } else -#endif { if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC) rq_len++; @@ -9867,12 +9264,10 @@ erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched, Uint len = 0; int no_rqs = erts_no_run_queues; -#ifdef ERTS_DIRTY_SCHEDULERS if (incl_dirty_io) no_rqs += ERTS_NUM_DIRTY_RUNQS; else no_rqs += ERTS_NUM_DIRTY_CPU_RUNQS; -#endif if (atomic_queues_read) { ERTS_ATOMIC_FOREACH_RUNQ_X(rq, no_rqs, @@ -9926,27 +9321,25 @@ erts_process_status(Process *rp, Eterm rpid) Process *p = rp ? rp : erts_proc_lookup_raw(rpid); if (p) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + erts_aint32_t state = erts_atomic32_read_acqb(&p->state); res = erts_process_state2status(state); } -#ifdef ERTS_SMP else { int i; ErtsSchedulerData *esdp; for (i = 0; i < erts_no_schedulers; i++) { esdp = ERTS_SCHEDULER_IX(i); - erts_smp_runq_lock(esdp->run_queue); + erts_runq_lock(esdp->run_queue); if (esdp->free_process && esdp->free_process->common.id == rpid) { res = am_free; - erts_smp_runq_unlock(esdp->run_queue); + erts_runq_unlock(esdp->run_queue); break; } - erts_smp_runq_unlock(esdp->run_queue); + erts_runq_unlock(esdp->run_queue); } } -#endif return res; } @@ -9962,9 +9355,9 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) int suspend; ASSERT(c_p == erts_get_current_process()); - ERTS_SMP_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(c_p_locks == erts_proc_lc_my_proc_locks(c_p)); if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); if (busy_port) suspend = erts_save_suspend_process_on_port(busy_port, c_p); @@ -9980,7 +9373,7 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) } if (!(c_p_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); if (suspend && busy_port && erts_system_monitor_flags.busy_port) monitor_generic(c_p, am_busy_port, busy_port->common.id); @@ -9989,12 +9382,12 @@ erts_suspend(Process* c_p, ErtsProcLocks c_p_locks, Port *busy_port) void erts_resume(Process* process, ErtsProcLocks process_locks) { - ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); + ERTS_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(process, ERTS_PROC_LOCK_STATUS); resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) - erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(process, ERTS_PROC_LOCK_STATUS); } int @@ -10014,7 +9407,7 @@ erts_resume_processes(ErtsProcList *list) resume_process(proc, ERTS_PROC_LOCK_STATUS); nresumed++; } - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); } fplp = plp; plp = plp->next; @@ -10026,7 +9419,7 @@ erts_resume_processes(ErtsProcList *list) Eterm erts_get_process_priority(Process *p) { - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_atomic32_read_nob(&p->state); switch (ERTS_PSFLGS_GET_USR_PRIO(state)) { case PRIORITY_MAX: return am_max; case PRIORITY_HIGH: return am_high; @@ -10049,7 +9442,7 @@ erts_set_process_priority(Process *p, Eterm value) default: return THE_NON_VALUE; break; } - a = erts_smp_atomic32_read_nob(&p->state); + a = erts_atomic32_read_nob(&p->state); if (nprio == ERTS_PSFLGS_GET_USR_PRIO(a)) oprio = nprio; else { @@ -10057,7 +9450,7 @@ erts_set_process_priority(Process *p, Eterm value) erts_aint32_t e, n, aprio; if (a & ERTS_PSFLG_ACTIVE_SYS) { - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); slocked = 1; } @@ -10071,7 +9464,7 @@ erts_set_process_priority(Process *p, Eterm value) int max_qbit; if (!slocked) { - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); slocked = 1; } @@ -10112,11 +9505,11 @@ erts_set_process_priority(Process *p, Eterm value) n |= ((nprio << ERTS_PSFLGS_USR_PRIO_OFFSET) | (aprio << ERTS_PSFLGS_ACT_PRIO_OFFSET)); - a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); } while (a != e); if (slocked) - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } @@ -10174,7 +9567,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ErtsRunQueue *rq; int context_reds; int fcalls; - int input_reductions; int actual_reds; int reds; Uint32 flags; @@ -10194,21 +9586,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (ERTS_USE_MODIFIED_TIMING()) { context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS; - input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS; } else { context_reds = CONTEXT_REDS; - input_reductions = INPUT_REDUCTIONS; } - ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()) + ERTS_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()) || !erts_thr_progress_is_blocking()); /* * Clean up after the process being scheduled out. */ if (!p) { /* NULL in the very first schedule() call */ -#ifdef ERTS_DIRTY_SCHEDULERS is_normal_sched = !esdp; if (is_normal_sched) { esdp = erts_get_scheduler_data(); @@ -10217,18 +9606,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) else { ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -#else - esdp = erts_get_scheduler_data(); - is_normal_sched = 1; -#endif rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; - erts_smp_runq_lock(rq); + erts_runq_lock(rq); } else { -#ifdef ERTS_SMP -#ifdef ERTS_DIRTY_SCHEDULERS is_normal_sched = !esdp; if (is_normal_sched) { esdp = p->scheduler_data; @@ -10237,21 +9619,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) else { ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -#else - esdp = p->scheduler_data; - is_normal_sched = 1; -#endif ASSERT(esdp->current_process == p || esdp->free_process == p); -#else - esdp = erts_scheduler_data; - ASSERT(esdp->current_process == p); - is_normal_sched = 1; -#endif sched_out_proc: - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); reds = actual_reds = calls - esdp->virtual_reds; @@ -10260,14 +9633,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); p->reds += actual_reds; - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); if (IS_TRACED(p)) { if (IS_TRACED_FL(p, F_TRACE_CALLS) && !(state & ERTS_PSFLG_FREE)) @@ -10286,20 +9658,17 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); -#ifdef ERTS_SMP if (p->trace_msg_q) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); erts_schedule_flush_trace_messages(p, 1); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); } -#endif /* have to re-read state after taking lock */ - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); -#ifdef ERTS_SMP if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_TRACE @@ -10308,7 +9677,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); -#endif esdp->reductions += reds; @@ -10326,18 +9694,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) actual_reds); esdp->current_process = NULL; -#ifdef ERTS_SMP if (is_normal_sched) p->scheduler_data = NULL; -#endif - erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + erts_proc_unlock(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS | ERTS_PROC_LOCK_TRACE)); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); -#ifdef ERTS_SMP if (state & ERTS_PSFLG_FREE) { if (!is_normal_sched) { ASSERT(p->flags & F_DELAYED_DEL_PROC); @@ -10347,44 +9712,40 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) esdp->free_process = NULL; } } -#endif if (dec_refc) erts_proc_dec_refc(p); } -#ifdef ERTS_SMP ASSERT(!esdp->free_process); -#endif ASSERT(!esdp->current_process); - ERTS_SMP_CHK_NO_PROC_LOCKS; - - if (is_normal_sched) { - if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) - (void) erts_get_monotonic_time(esdp); + ERTS_CHK_NO_PROC_LOCKS; - if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - erts_smp_runq_unlock(rq); - erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); - erts_smp_runq_lock(rq); - } - } } - ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); check_activities_to_run: { erts_aint32_t psflg_running, psflg_running_sys; -#ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; if (is_normal_sched) { + + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); + + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_runq_lock(rq); + } + if (rq->check_balance_reds <= 0) check_balance(rq); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); mps = erts_get_migration_paths_managed(); mp = &mps->mpath[rq->ix]; @@ -10393,14 +9754,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) immigrate(rq, mp); } - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); continue_check_activities_to_run: flags = ERTS_RUNQ_FLGS_GET_NOB(rq); continue_check_activities_to_run_known_flags: ASSERT(!is_normal_sched || (flags & ERTS_RUNQ_FLG_NONEMPTY)); if (!is_normal_sched) { - if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + if (erts_atomic32_read_acqb(&esdp->ssi->flags) & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { suspend_scheduler(esdp); } @@ -10430,26 +9791,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); if (aux_work | leader_update) { - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); } - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); -#else /* ERTS_SMP */ - { - erts_aint32_t aux_work; - aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work) - handle_aux_work(&esdp->aux_work_data, aux_work, 0); - } -#endif /* ERTS_SMP */ flags = ERTS_RUNQ_FLGS_GET_NOB(rq); @@ -10460,8 +9813,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } else if (!runq_got_work_to_execute_flags(flags)) { /* Prepare for scheduler wait */ -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; @@ -10475,7 +9827,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ASSERT(!runq_got_work_to_execute(rq)); if (!is_normal_sched) { /* Dirty scheduler */ - if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + if (erts_atomic32_read_acqb(&esdp->ssi->flags) & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { /* Go suspend... */ goto continue_check_activities_to_run_known_flags; @@ -10491,7 +9843,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) */ flags = ERTS_RUNQ_FLGS_GET_NOB(rq); if ((flags & ERTS_RUNQ_FLG_SUSPENDED) -#ifdef ERTS_DIRTY_SCHEDULERS /* If multi scheduling block and we have * dirty work, suspend and let dirty * scheduler handle work... */ @@ -10499,7 +9850,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_RUNQ_FLG_MSB_EXEC)) == ERTS_RUNQ_FLG_MSB_EXEC)) && have_dirty_work()) -#endif ) { non_empty_runq(rq); flags |= ERTS_RUNQ_FLG_NONEMPTY; @@ -10511,62 +9861,21 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } empty_runq(rq); } -#endif (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC); scheduler_wait(&fcalls, esdp, rq); flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); flags |= ERTS_RUNQ_FLG_EXEC; ERTS_MSACC_UPDATE_CACHE(); -#ifdef ERTS_SMP non_empty_runq(rq); -#endif - - goto check_activities_to_run; - } - else if (is_normal_sched - && (fcalls > input_reductions - && prepare_for_sys_schedule(!0))) { - ErtsMonotonicTime current_time; - /* - * Schedule system-level activities. - */ - - ERTS_MSACC_PUSH_STATE_CACHED_M(); - - erts_smp_atomic32_set_relb(&function_calls, 0); - fcalls = 0; - -#if 0 /* Not needed since we wont wait in sys schedule */ - erts_sys_schedule_interrupt(0); -#endif - erts_smp_runq_unlock(rq); - - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); - LTTNG2(scheduler_poll, esdp->no, 1); - - erl_sys_schedule(1); - ERTS_MSACC_POP_STATE_M(); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); - -#ifdef ERTS_SMP - erts_smp_runq_lock(rq); - clear_sys_scheduling(); - goto continue_check_activities_to_run; -#else goto check_activities_to_run; -#endif } if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); -#ifdef ERTS_SMP wakeup_other.check(rq, flags); -#endif /* * Find a new port to run. @@ -10575,29 +9884,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); if (flags & PORT_BIT) { - int have_outstanding_io; - have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((!erts_eager_check_io - && have_outstanding_io - && fcalls > 2*input_reductions) - || (flags & ERTS_RUNQ_FLG_HALTING)) { - /* - * If we have performed more than 2*INPUT_REDUCTIONS since - * last call to erl_sys_schedule() and we still haven't - * handled all I/O tasks we stop running processes and - * focus completely on ports. - * - * One could argue that this is a strange behavior. The - * reason for doing it this way is that it is similar - * to the behavior before port tasks were introduced. - * We don't want to change the behavior too much, at - * least not at the time of writing. This behavior - * might change in the future. - * - * /rickard - */ - goto check_activities_to_run; - } + erts_port_task_execute(rq, &esdp->current_port); + if (flags & ERTS_RUNQ_FLG_HALTING) + goto check_activities_to_run; } /* @@ -10664,13 +9953,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) proxy_p = NULL; goto pick_next_process; } - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); } -#ifdef ERTS_DIRTY_SCHEDULERS if (!is_normal_sched) clear_proc_dirty_queue_bit(p, rq, qbit); -#endif while (1) { erts_aint32_t exp, new; @@ -10688,7 +9975,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_FREE))) -#ifdef ERTS_DIRTY_SCHEDULERS | (((state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_FREE @@ -10697,7 +9983,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) & (!!is_normal_sched)) -#endif ) & ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_EXITING @@ -10706,11 +9991,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) != ERTS_PSFLG_SUSPENDED) -#ifdef ERTS_DIRTY_SCHEDULERS & (!(state & (ERTS_PSFLG_EXITING | ERTS_PSFLG_PENDING_EXIT)) | (!!is_normal_sched)) -#endif ); if (run_process) { @@ -10720,7 +10003,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) else new |= psflg_running; } - state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp); + state = erts_atomic32_cmpxchg_relb(&p->state, new, exp); if (state == exp) { if (!run_process) { if (proxy_p) { @@ -10747,22 +10030,21 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) calls = 0; reds = context_reds; - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); -#ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); if (erts_sched_stat.enabled) { int prio; @@ -10773,28 +10055,24 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state); - erts_smp_spin_lock(&erts_sched_stat.lock); + erts_spin_lock(&erts_sched_stat.lock); erts_sched_stat.prio[prio].total_executed++; erts_sched_stat.prio[prio].executed++; if (migrated) { erts_sched_stat.prio[prio].total_migrated++; erts_sched_stat.prio[prio].migrated++; } - erts_smp_spin_unlock(&erts_sched_stat.lock); + erts_spin_unlock(&erts_sched_stat.lock); } - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); -#ifndef ERTS_DIRTY_SCHEDULERS - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; -#else /* ERTS_DIRTY_SCHEDULERS */ if (is_normal_sched) { if ((!!(state & ERTS_PSFLGS_DIRTY_WORK)) & (!(state & ERTS_PSFLG_ACTIVE_SYS))) { /* Migrate to dirty scheduler... */ sunlock_sched_out_proc: - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); goto sched_out_proc; } ASSERT(!p->scheduler_data); @@ -10824,17 +10102,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) : (rq == ERTS_DIRTY_IO_RUNQ && (state & ERTS_PSFLG_DIRTY_IO_PROC))); } -#endif if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); - state = erts_smp_atomic32_read_nob(&p->state); + state = erts_atomic32_read_nob(&p->state); } -#endif /* ERTS_SMP */ - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); /* Clear tracer if it has been removed */ if (IS_TRACED(p) && erts_is_tracer_proc_enabled( @@ -10870,10 +10146,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds -= cost; if (reds <= 0) goto sched_out_proc; -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLGS_DIRTY_WORK) goto sched_out_proc; -#endif } ASSERT(state & psflg_running_sys); @@ -10892,7 +10166,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) n &= ~psflg_running_sys; n |= psflg_running; - state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + state = erts_atomic32_cmpxchg_mb(&p->state, n, e); if (state == e) { state = n; break; @@ -10911,10 +10185,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) reds -= cost; if (reds <= 0) goto sched_out_proc; -#ifdef ERTS_DIRTY_SCHEDULERS if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) goto sched_out_proc; -#endif } } } @@ -10926,12 +10198,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->fcalls = reds; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ #ifdef DEBUG { - erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t dstate = erts_atomic32_read_nob(&p->state); ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate) || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate)); } @@ -10941,9 +10213,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) { BeamInstr** pi; -#ifdef ERTS_SMP ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); -#endif pi = (BeamInstr **) p->def_arg_reg; p->i = *pi; p->flags &= ~F_INSLPQUEUE; @@ -10960,13 +10230,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result, int normal_sched) { Process *rp; -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal_sched) rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, st->requester, 0, ERTS_P2P_FLG_INC_REFC); else -#endif rp = erts_proc_lookup(st->requester); if (rp) { ErtsProcLocks rp_locks; @@ -11014,12 +10282,10 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); -#ifdef ERTS_DIRTY_SCHEDULERS if (!normal_sched) erts_proc_dec_refc(rp); -#endif } erts_cleanup_offheap(&st->off_heap); @@ -11038,7 +10304,7 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop) *priop = -1; /* Shut up annoying erroneous warning */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); if (!c_p->sys_task_qs) { qmask = 0; @@ -11158,13 +10424,13 @@ fetch_sys_task(Process *c_p, erts_aint32_t state, int *qmaskp, int *priop) if (a == n) break; - a = erts_smp_atomic32_cmpxchg_nob(&c_p->state, n, e); + a = erts_atomic32_cmpxchg_nob(&c_p->state, n, e); } while (a != e); } done: - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); if (unused_qs) proc_sys_task_queues_free(unused_qs); @@ -11175,9 +10441,7 @@ done: } static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); -#ifdef ERTS_DIRTY_SCHEDULERS static void save_dirty_task(Process *c_p, ErtsProcSysTask *st); -#endif static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) @@ -11188,7 +10452,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) int qmask = 0; ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); do { ErtsProcSysTaskType type; @@ -11197,10 +10461,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) Eterm st_res; if (state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) { -#ifdef ERTS_SMP if (state & ERTS_PSFLG_PENDING_EXIT) erts_handle_pending_exit(c_p, ERTS_PROC_LOCK_MAIN); -#endif ASSERT(ERTS_PROC_IS_EXITING(c_p)); break; } @@ -11227,13 +10489,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) FLAGS(c_p) |= F_NEED_FULLSWEEP; } reds -= scheduler_gc_proc(c_p, reds); -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { save_dirty_task(c_p, st); st = NULL; break; } -#endif if (type == ERTS_PSTT_GC_MAJOR) minor_gc = major_gc = 1; else @@ -11275,13 +10535,11 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) fcalls, do_gc); reds -= cla_reds; if (is_non_value(st_res)) { -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->flags & F_DIRTY_CLA) { save_dirty_task(c_p, st); st = NULL; break; } -#endif /* Needed gc, but gc was disabled */ save_gc_task(c_p, st, st_prio); st = NULL; @@ -11295,18 +10553,14 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; break; -#ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); st_res = am_true; break; -#endif -#ifdef ERTS_SMP case ERTS_PSTT_ETS_FREE_FIXATION: reds -= erts_db_execute_free_fixation(c_p, (DbFixation*)st->arg[0]); st_res = am_true; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -11315,7 +10569,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) if (st) reds += notify_sys_task_executed(c_p, st, st_res, 1); - state = erts_smp_atomic32_read_acqb(&c_p->state); + state = erts_atomic32_read_acqb(&c_p->state); } while (qmask && reds > 0); *statep = state; @@ -11336,20 +10590,18 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) * are dirty tasks. */ - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); do { ErtsProcSysTask *st; Eterm st_res; int st_prio; -#ifdef ERTS_DIRTY_SCHEDULERS if (c_p->dirty_sys_tasks) { st = c_p->dirty_sys_tasks; c_p->dirty_sys_tasks = st->next; } else -#endif { st = fetch_sys_task(c_p, state, &qmask, &st_prio); if (!st) @@ -11367,12 +10619,10 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_CLA: st_res = am_ok; break; -#ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); st_res = am_true; break; -#endif default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -11381,13 +10631,12 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) reds += notify_sys_task_executed(c_p, st, st_res, 1); - state = erts_smp_atomic32_read_acqb(&c_p->state); + state = erts_atomic32_read_acqb(&c_p->state); } while (qmask && reds < max_reds); return reds; } -#ifdef ERTS_DIRTY_SCHEDULERS void erts_execute_dirty_system_task(Process *c_p) @@ -11416,19 +10665,19 @@ erts_execute_dirty_system_task(Process *c_p) } if (c_p->flags & F_DIRTY_GC_HIBERNATE) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); if (c_p->msg.len) c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */ else { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); c_p->fvalue = NIL; erts_garbage_collect_hibernate(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); } if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { @@ -11472,7 +10721,7 @@ erts_execute_dirty_system_task(Process *c_p) } - erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); + erts_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); } static BIF_RETTYPE @@ -11525,12 +10774,11 @@ dispatch_system_task(Process *c_p, erts_aint_t fail_state, erts_queue_message(rp, rp_locks, mp, msg, st->requester); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); return ret; } -#endif static BIF_RETTYPE request_system_task(Process *c_p, Eterm requester, Eterm target, @@ -11635,7 +10883,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, st->type = ERTS_PSTT_CPC; if (!rp) goto noproc; -#ifdef ERTS_DIRTY_SCHEDULERS /* * If the process should start executing dirty * code it is important that this task is @@ -11643,7 +10890,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, */ fail_state |= (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS); -#endif break; case am_copy_literals: @@ -11665,14 +10911,12 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, noproc: failure = noproc_res; } -#ifdef ERTS_DIRTY_SCHEDULERS else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { ret = dispatch_system_task(c_p, fail_state, st, target, priority, operation); goto cleanup_return; } -#endif else { ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); failure = am_internal_error; @@ -11688,9 +10932,7 @@ badarg: ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); -#ifdef ERTS_DIRTY_SCHEDULERS cleanup_return: -#endif if (st) { erts_cleanup_offheap(&st->off_heap); @@ -11731,7 +10973,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type, void* arg) st->req_id_sz = 0; st->arg[0] = (Eterm)arg; ERTS_INIT_OFF_HEAP(&st->off_heap); - state = erts_smp_atomic32_read_nob(&rp->state); + state = erts_atomic32_read_nob(&rp->state); fail_state = ERTS_PSFLG_EXITING; @@ -11754,7 +10996,6 @@ erts_schedule_ets_free_fixation(Eterm pid, DbFixation* fix) erts_schedule_generic_sys_task(pid, ERTS_PSTT_ETS_FREE_FIXATION, fix); } -#ifdef ERTS_DIRTY_SCHEDULERS static void flush_dirty_trace_messages(void *vpid) @@ -11768,46 +11009,38 @@ flush_dirty_trace_messages(void *vpid) erts_free(ERTS_ALC_T_DIRTY_SL, vpid); #endif - proc = erts_proc_lookup(pid); - if (proc) - (void) erts_flush_trace_messages(proc, 0); + proc = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_MAIN, 0); + if (proc) { + (void) erts_flush_trace_messages(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + } } -#endif /* ERTS_DIRTY_SCHEDULERS */ void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) { -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl; -#endif Eterm pid = proc->common.id; -#ifdef ERTS_DIRTY_SCHEDULERS erts_aint32_t state; if (!force_on_proc) { - state = erts_smp_atomic32_read_nob(&proc->state); + state = erts_atomic32_read_nob(&proc->state); if (state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { goto sched_flush_dirty; } } -#endif -#ifdef ERTS_SMP dhndl = erts_thr_progress_unmanaged_delay(); -#endif erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ, NULL); -#ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); -#endif -#ifdef ERTS_DIRTY_SCHEDULERS if (!force_on_proc) { - state = erts_smp_atomic32_read_mb(&proc->state); + state = erts_atomic32_read_mb(&proc->state); if (state & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { void *vargp; @@ -11835,7 +11068,6 @@ erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp); } } -#endif } static void @@ -11844,7 +11076,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) erts_aint32_t state; ErtsProcSysTaskQs *qs; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); qs = ERTS_PROC_GET_DELAYED_GC_TASK_QS(c_p); if (!qs) { @@ -11874,7 +11106,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) } } - state = erts_smp_atomic32_read_nob(&c_p->state); + state = erts_atomic32_read_nob(&c_p->state); ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_DIRTY_RUNNING @@ -11890,20 +11122,18 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; n |= prio << ERTS_PSFLGS_ACT_PRIO_OFFSET; } - state = erts_smp_atomic32_cmpxchg_relb(&c_p->state, n, e); + state = erts_atomic32_cmpxchg_relb(&c_p->state, n, e); if (state == e) break; } } -#ifdef ERTS_DIRTY_SCHEDULERS static void save_dirty_task(Process *c_p, ErtsProcSysTask *st) { st->next = c_p->dirty_sys_tasks; c_p->dirty_sys_tasks = st; } -#endif int erts_set_gc_state(Process *c_p, int enable) @@ -11911,8 +11141,8 @@ erts_set_gc_state(Process *c_p, int enable) ErtsProcSysTaskQs *dgc_tsk_qs; ASSERT(c_p == erts_get_current_process()); ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - & erts_smp_atomic32_read_nob(&c_p->state)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); + & erts_atomic32_read_nob(&c_p->state)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); if (!enable) { c_p->flags |= F_DISABLE_GC; @@ -11927,7 +11157,7 @@ erts_set_gc_state(Process *c_p, int enable) /* Move delayed gc tasks into sys tasks queues. */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); if (!c_p->sys_task_qs) { c_p->sys_task_qs = dgc_tsk_qs; @@ -12000,7 +11230,7 @@ erts_set_gc_state(Process *c_p, int enable) erts_aint32_t aprio, state = #endif - erts_smp_atomic32_read_bset_nob(&c_p->state, + erts_atomic32_read_bset_nob(&c_p->state, (ERTS_PSFLG_DELAYED_SYS | ERTS_PSFLG_ACTIVE_SYS), ERTS_PSFLG_ACTIVE_SYS); @@ -12014,7 +11244,7 @@ erts_set_gc_state(Process *c_p, int enable) } #endif - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL); @@ -12030,24 +11260,24 @@ erts_sched_stat_modify(int what) int ix; switch (what) { case ERTS_SCHED_STAT_MODIFY_ENABLE: - erts_smp_thr_progress_block(); + erts_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_DISABLE: - erts_smp_thr_progress_block(); + erts_thr_progress_block(); erts_sched_stat.enabled = 0; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: - erts_smp_spin_lock(&erts_sched_stat.lock); + erts_spin_lock(&erts_sched_stat.lock); for (ix = 0; ix < ERTS_NO_PRIO_LEVELS; ix++) { erts_sched_stat.prio[ix].total_executed = 0; erts_sched_stat.prio[ix].executed = 0; erts_sched_stat.prio[ix].total_migrated = 0; erts_sched_stat.prio[ix].migrated = 0; } - erts_smp_spin_unlock(&erts_sched_stat.lock); + erts_spin_unlock(&erts_sched_stat.lock); break; } } @@ -12061,7 +11291,7 @@ erts_sched_stat_term(Process *p, int total) Uint executed[ERTS_NO_PRIO_LEVELS]; Uint migrated[ERTS_NO_PRIO_LEVELS]; - erts_smp_spin_lock(&erts_sched_stat.lock); + erts_spin_lock(&erts_sched_stat.lock); if (total) { int i; for (i = 0; i < ERTS_NO_PRIO_LEVELS; i++) { @@ -12080,7 +11310,7 @@ erts_sched_stat_term(Process *p, int total) erts_sched_stat.prio[i].migrated = 0; } } - erts_smp_spin_unlock(&erts_sched_stat.lock); + erts_spin_unlock(&erts_sched_stat.lock); sz = 0; (void) erts_bld_atom_2uint_3tup_list(NULL, &sz, ERTS_NO_PRIO_LEVELS, @@ -12100,7 +11330,6 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); ErtsMiscOpList *molp = misc_op_list_alloc(); -#ifdef ERTS_SMP ErtsMigrationPaths *mpaths = erts_get_migration_paths(); if (!mpaths) @@ -12110,9 +11339,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) if (erq) rq = erq; } -#endif - erts_smp_runq_lock(rq); + erts_runq_lock(rq); molp->next = NULL; molp->func = func; @@ -12123,13 +11351,11 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) rq->misc.start = molp; rq->misc.end = molp; -#ifdef ERTS_SMP non_empty_runq(rq); -#endif ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); smp_notify_inc_runq(rq); } @@ -12162,7 +11388,7 @@ exec_misc_ops(ErtsRunQueue *rq) if (!rq->misc.start) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); while (molp) { tmp_molp = molp; @@ -12171,7 +11397,7 @@ exec_misc_ops(ErtsRunQueue *rq) misc_op_list_free(tmp_molp); } - erts_smp_runq_lock(rq); + erts_runq_lock(rq); } Uint @@ -12202,12 +11428,12 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) { Uint reds = erts_current_reductions(c_p, c_p); int ix; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); /* * Wait for other schedulers to schedule out their processes * and update 'reductions'. */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++) reds += ERTS_RUNQ_IX(ix)->procs.reductions; if (redsp) @@ -12215,8 +11441,8 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) if (diffp) *diffp = reds - last_exact_reductions; last_exact_reductions = reds; - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_thr_progress_unblock(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } static void delete_process(Process* p); @@ -12224,10 +11450,8 @@ static void delete_process(Process* p); void erts_free_proc(Process *p) { -#ifdef ERTS_SMP erts_proc_lock_fin(p); -#endif - ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); + ASSERT(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); ASSERT(0 == erts_proc_read_refc(p)); if (p->flags & F_DELAYED_DEL_PROC) delete_process(p); @@ -12246,17 +11470,13 @@ static void early_init_process_struct(void *varg, Eterm data) Process *proc = arg->proc; proc->common.id = make_internal_pid(data); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_init_nob(&proc->dirty_state, 0); + erts_atomic32_init_nob(&proc->dirty_state, 0); proc->dirty_sys_tasks = NULL; -#endif - erts_smp_atomic32_init_relb(&proc->state, arg->state); + erts_atomic32_init_relb(&proc->state, arg->state); -#ifdef ERTS_SMP RUNQ_SET_RQ(&proc->run_queue, arg->run_queue); erts_proc_lock_init(proc); /* All locks locked */ -#endif } @@ -12329,7 +11549,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif - erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); /* * Check for errors. @@ -12377,9 +11597,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } - ASSERT((erts_smp_atomic32_read_nob(&p->state) + ASSERT((erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_ON_HEAP_MSGQ) - || (erts_smp_atomic32_read_nob(&p->state) + || (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ)); #ifdef SHCOPY_SPAWN @@ -12405,7 +11625,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_vheap_size = BIN_VH_MIN_SIZE; MAX_HEAP_SIZE_SET(p, H_MAX_SIZE); MAX_HEAP_SIZE_FLAGS_SET(p, H_MAX_FLAGS); - p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_atomic32_read_nob(&erts_max_gen_gcs); } p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -12431,10 +11651,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef HIPE hipe_init_process(&p->hipe); -#ifdef ERTS_SMP hipe_init_process_smp(&p->hipe_smp); #endif -#endif p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz); p->old_hend = p->old_htop = p->old_heap = NULL; p->high_water = p->heap; @@ -12501,17 +11719,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg.first = NULL; p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; + p->msg.saved_last = &p->msg.first; p->msg.len = 0; -#ifdef ERTS_SMP p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; -#endif p->bif_timers = NULL; p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; - erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); + erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; @@ -12529,14 +11746,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->last_old_htop = NULL; #endif -#ifdef ERTS_SMP p->trace_msg_q = NULL; p->scheduler_data = NULL; p->suspendee = NIL; p->pending_suspenders = NULL; p->pending_exit.reason = THE_NON_VALUE; p->pending_exit.bp = NULL; -#endif #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -12564,8 +11779,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); if (so->flags & SPO_LINK) trace_proc(parent, locks, parent, am_link, p->common.id); @@ -12577,8 +11792,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { /* This happens when parent was not traced, but child is */ locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_smp_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); } trace_proc_spawn(p, am_spawned, parent->common.id, mod, func, args); if (so->flags & SPO_LINK) @@ -12617,7 +11832,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). so->mref = mref; } - erts_smp_proc_unlock(p, locks); + erts_proc_unlock(p, locks); res = p->common.id; @@ -12625,7 +11840,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ - erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); schedule_process(p, state, 0); @@ -12645,7 +11860,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). error: - erts_smp_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(parent, locks & ERTS_PROC_LOCKS_ALL_MINOR); return res; } @@ -12696,7 +11911,7 @@ void erts_init_empty_process(Process *p) p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; - erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); + erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL); ERTS_P_MONITORS(p) = NULL; ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; @@ -12739,23 +11954,18 @@ void erts_init_empty_process(Process *p) #ifdef HIPE hipe_init_process(&p->hipe); -#ifdef ERTS_SMP hipe_init_process_smp(&p->hipe_smp); #endif -#endif INIT_HOLE_CHECK(p); #ifdef DEBUG p->last_old_htop = NULL; #endif -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_init_nob(&p->dirty_state, 0); + erts_atomic32_init_nob(&p->dirty_state, 0); p->dirty_sys_tasks = NULL; -#endif - erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); + erts_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); -#ifdef ERTS_SMP p->scheduler_data = NULL; p->msg_inq.first = NULL; p->msg_inq.last = &p->msg_inq.first; @@ -12765,9 +11975,8 @@ void erts_init_empty_process(Process *p) p->pending_exit.reason = THE_NON_VALUE; p->pending_exit.bp = NULL; erts_proc_lock_init(p); - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0)); -#endif #if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; @@ -12814,14 +12023,12 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->parent == NIL); -#ifdef ERTS_SMP ASSERT(p->msg_inq.first == NULL); ASSERT(p->msg_inq.len == 0); ASSERT(p->suspendee == NIL); ASSERT(p->pending_suspenders == NULL); ASSERT(p->pending_exit.reason == THE_NON_VALUE); ASSERT(p->pending_exit.bp == NULL); -#endif /* Thing that erts_cleanup_empty_process() cleans up */ @@ -12846,9 +12053,7 @@ erts_cleanup_empty_process(Process* p) free_message_buffer(p->mbuf); p->mbuf = NULL; } -#ifdef ERTS_SMP erts_proc_lock_fin(p); -#endif #ifdef DEBUG erts_debug_verify_clean_empty_process(p); #endif @@ -12880,10 +12085,10 @@ delete_process(Process* p) /* Cleanup psd */ - psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd); if (psd) { - erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */ + erts_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */ erts_free(ERTS_ALC_T_PSD, psd); } @@ -12941,7 +12146,7 @@ set_proc_exiting(Process *p, { erts_aint32_t state = in_state, enq_prio = -1; int enqueue; - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); enqueue = change_proc_schedule_state(p, (ERTS_PSFLG_SUSPENDED @@ -12964,22 +12169,6 @@ set_proc_exiting(Process *p, KILL_CATCHES(p); p->i = (BeamInstr *) beam_exit; -#ifndef ERTS_SMP - if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - && !(state & ERTS_PSFLG_GC)) { - /* - * I non smp case: - * - * Currently executing process might be sent an exit - * signal if it is traced by a port that it also is - * linked to, and the port terminates during the - * trace. In this case we want schedule out the - * process as quickly as possible in order to detect - * the event as fast as possible. - */ - ERTS_VBUMP_ALL_REDS(p); - } -#endif add2runq(enqueue, enq_prio, p, state, NULL); } @@ -12992,9 +12181,9 @@ set_proc_self_exiting(Process *c_p) #endif erts_aint32_t state, enq_prio = -1; - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL); - state = erts_smp_atomic32_read_nob(&c_p->state); + state = erts_atomic32_read_nob(&c_p->state); ASSERT(state & (ERTS_PSFLG_RUNNING |ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_DIRTY_RUNNING @@ -13014,38 +12203,37 @@ set_proc_self_exiting(Process *c_p) return state; } -#ifdef ERTS_SMP void erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) { ErtsProcLocks xlocks; ASSERT(is_value(c_p->pending_exit.reason)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); - ERTS_SMP_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); - ERTS_SMP_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) - & erts_smp_atomic32_read_nob(&c_p->state))); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == locks); + ERTS_LC_ASSERT(locks & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(!((ERTS_PSFLG_EXITING|ERTS_PSFLG_FREE) + & erts_atomic32_read_nob(&c_p->state))); /* Ensure that all locks on c_p are locked before proceeding... */ if (locks == ERTS_PROC_LOCKS_ALL) xlocks = 0; else { xlocks = ~locks & ERTS_PROC_LOCKS_ALL; - if (erts_smp_proc_trylock(c_p, xlocks) == EBUSY) { - erts_smp_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (erts_proc_trylock(c_p, xlocks) == EBUSY) { + erts_proc_unlock(c_p, locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } } set_proc_exiting(c_p, - erts_smp_atomic32_read_acqb(&c_p->state), + erts_atomic32_read_acqb(&c_p->state), c_p->pending_exit.reason, c_p->pending_exit.bp); c_p->pending_exit.reason = THE_NON_VALUE; c_p->pending_exit.bp = NULL; if (xlocks) - erts_smp_proc_unlock(c_p, xlocks); + erts_proc_unlock(c_p, xlocks); } static void save_pending_exiter(Process *p, ErtsProcList *plp); @@ -13066,9 +12254,9 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs) * pending exit will soon be detected and handled by the * scheduler running the process (at schedule in/out). */ - if (erts_smp_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) { + if (erts_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) { if (erts_proclist_same(plp, p)) { - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_EXITING))) { @@ -13076,12 +12264,12 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs) erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); } } - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); } else { - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); if (erts_proclist_same(plp, p)) { - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_EXITING))) { @@ -13093,7 +12281,7 @@ do_handle_pending_exiters(ErtsProcList *pnd_xtrs) plp = NULL; } } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } } if (plp) @@ -13108,7 +12296,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp) ErtsSchedulerSleepInfo *ssi; ErtsRunQueue *rq; - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); rq = RUNQ_READ_RQ(&p->run_queue); ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); @@ -13116,7 +12304,7 @@ save_pending_exiter(Process *p, ErtsProcList *plp) if (!plp) plp = proclist_create(p); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); erts_proclist_store_last(&rq->procs.pending_exiters, plp); @@ -13124,12 +12312,11 @@ save_pending_exiter(Process *p, ErtsProcList *plp) ssi = rq->scheduler->ssi; - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); } -#endif /* * This function delivers an EXIT message to a process @@ -13265,11 +12452,11 @@ send_exit_signal(Process *c_p, /* current process if and only Uint32 flags /* flags */ ) { - erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + erts_aint32_t state = erts_atomic32_read_nob(&rp->state); Eterm rsn = reason == am_kill ? am_killed : reason; - ERTS_SMP_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); - ERTS_SMP_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) + ERTS_LC_ASSERT(*rp_locks == erts_proc_lc_my_proc_locks(rp)); + ERTS_LC_ASSERT((*rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) == ERTS_PROC_LOCKS_XSIG_SEND); ASSERT(reason != THE_NON_VALUE); @@ -13290,7 +12477,7 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { /* have to release the status lock in order to send the exit message */ - erts_smp_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_unlock(rp, *rp_locks & ERTS_PROC_LOCKS_XSIG_SEND); *rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); @@ -13301,7 +12488,6 @@ send_exit_signal(Process *c_p, /* current process if and only return 1; /* Receiver will get a message */ } else if (reason != am_normal || (flags & ERTS_XSIG_FLG_NO_IGN_NORMAL)) { -#ifdef ERTS_SMP if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT))) { ASSERT(!rp->pending_exit.bp); @@ -13311,10 +12497,10 @@ send_exit_signal(Process *c_p, /* current process if and only if (*rp_locks != ERTS_PROC_LOCKS_ALL) { ErtsProcLocks need_locks = (~(*rp_locks) & ERTS_PROC_LOCKS_ALL); - if (erts_smp_proc_trylock(c_p, need_locks) == EBUSY) { - erts_smp_proc_unlock(c_p, + if (erts_proc_trylock(c_p, need_locks) == EBUSY) { + erts_proc_unlock(c_p, *rp_locks & ~ERTS_PROC_LOCK_MAIN); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } *rp_locks = ERTS_PROC_LOCKS_ALL; } @@ -13328,7 +12514,7 @@ send_exit_signal(Process *c_p, /* current process if and only ErlHeapFragment *bp = NULL; Eterm rsn_cpy; if (need_locks - && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + && erts_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ save_pending_exiter(rp, NULL); /* @@ -13340,7 +12526,7 @@ send_exit_signal(Process *c_p, /* current process if and only /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; - state = erts_smp_atomic32_read_nob(&rp->state); + state = erts_atomic32_read_nob(&rp->state); if (is_immed(rsn)) rsn_cpy = rsn; @@ -13348,14 +12534,12 @@ send_exit_signal(Process *c_p, /* current process if and only Eterm *hp; ErlOffHeap *ohp; Uint rsn_sz = size_object(rsn); -#ifdef ERTS_DIRTY_SCHEDULERS if (state & ERTS_PSFLG_DIRTY_RUNNING) { bp = new_message_buffer(rsn_sz); ohp = &bp->off_heap; hp = &bp->mem[0]; } else -#endif { hp = HAlloc(rp, rsn_sz); ohp = &rp->off_heap; @@ -13395,12 +12579,8 @@ send_exit_signal(Process *c_p, /* current process if and only * has been scheduled, we may need to add it to a normal run * queue... */ -#ifndef ERTS_DIRTY_SCHEDULERS - (void) erts_smp_atomic32_read_bor_relb(&rp->state, - ERTS_PSFLG_PENDING_EXIT); -#else { - erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state); + erts_aint32_t a = erts_atomic32_read_nob(&rp->state); while (1) { erts_aint32_t n, e; int dwork; @@ -13408,7 +12588,7 @@ send_exit_signal(Process *c_p, /* current process if and only n |= ERTS_PSFLG_PENDING_EXIT; dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK); n &= ~ERTS_PSFLGS_DIRTY_WORK; - a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e); + a = erts_atomic32_cmpxchg_mb(&rp->state, n, e); if (a == e) { if (dwork) erts_schedule_process(rp, n, *rp_locks); @@ -13416,7 +12596,6 @@ send_exit_signal(Process *c_p, /* current process if and only } } } -#endif } } /* else: @@ -13428,17 +12607,6 @@ send_exit_signal(Process *c_p, /* current process if and only * that the receiver *will* exit; either on the pending * exit or by itself before seeing the pending exit. */ -#else /* !ERTS_SMP */ - erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); - if (!(state & ERTS_PSFLG_EXITING)) { - set_proc_exiting(rp, - state, - (is_immed(rsn) || c_p == rp - ? rsn - : copy_object(rsn, rp)), - NULL); - } -#endif return -1; /* Receiver will exit */ } @@ -13486,9 +12654,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ASSERT(is_node_name_atom(mon->u.pid)); dep = erts_sysname_to_connected_dist_entry(mon->u.pid); if (dep) { - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; int code = erts_dsig_prepare(&dsd, dep, NULL, @@ -13503,7 +12671,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } erts_destroy_monitor(rmon); } - erts_deref_dist_entry(dep); } } else { ASSERT(is_pid(mon->u.pid) || is_port(mon->u.pid)); @@ -13514,7 +12681,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) goto done; } rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon == NULL) { goto done; } @@ -13533,9 +12700,9 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) dep = external_pid_dist_entry(mon->u.pid); ASSERT(dep != NULL); if (dep) { - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; int code = erts_dsig_prepare(&dsd, dep, NULL, @@ -13586,15 +12753,15 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } UnUseTmpHeapNoproc(3); /* else: demonitor while we exited, i.e. do nothing... */ - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } else { /* external by pid or name */ ASSERT(is_external_pid(mon->u.pid)); dep = external_pid_dist_entry(mon->u.pid); ASSERT(dep != NULL); if (dep) { - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); rmon = erts_remove_monitor(&(dep->monitors), mon->ref); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (rmon) { ErtsDSigData dsd; int code = erts_dsig_prepare(&dsd, dep, NULL, @@ -13702,7 +12869,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) /* We didn't exit the process and it is traced */ if (IS_TRACED_FL(rp, F_TRACE_PROCS)) { if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; } trace_proc(NULL, 0, rp, am_getting_unlinked, p->common.id); @@ -13710,7 +12877,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) } } ASSERT(rp != p); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } else if (is_external_pid(item)) { @@ -13720,14 +12887,14 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) int code; ErtsDistLinkData dld; erts_remove_dist_link(&dld, p->common.id, item, dep); - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); code = erts_dsig_prepare(&dsd, dep, p, ERTS_DSP_NO_LOCK, 0); if (code == ERTS_DSIG_PREP_CONNECTED) { code = erts_dsig_send_exit_tt(&dsd, p->common.id, item, reason, SEQ_TRACE_TOKEN(p)); ASSERT(code == ERTS_DSIG_SEND_OK); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_destroy_dist_link(&dld); } } @@ -13738,12 +12905,11 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) if(dep) { /* dist entries have node links in a separate structure to avoid confusion */ - erts_smp_de_links_lock(dep); + erts_de_links_lock(dep); rlnk = erts_remove_link(&(dep->node_links), p->common.id); - erts_smp_de_links_unlock(dep); + erts_de_links_unlock(dep); if (rlnk) erts_destroy_link(rlnk); - erts_deref_dist_entry(dep); } break; @@ -13763,7 +12929,7 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) ASSERT(suspendee != vc_p); if (smon->active) resume_process(suspendee, ERTS_PROC_LOCK_STATUS); - erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } erts_destroy_suspend_monitor(smon); } @@ -13792,18 +12958,13 @@ erts_do_exit_process(Process* p, Eterm reason) erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", p->common.id, reason); -#ifdef ERTS_SMP - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going to exiting state (ERTS_PSFLG_EXITING), it is enough to take any lock when looking up a process (erts_pid2proc()) to prevent the looked up process from exiting until the lock has been released. */ - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#endif + erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); -#ifndef ERTS_SMP - set_proc_self_exiting(p); -#else if (ERTS_PSFLG_PENDING_EXIT & set_proc_self_exiting(p)) { /* Process exited before pending exit was received... */ p->pending_exit.reason = THE_NON_VALUE; @@ -13815,8 +12976,7 @@ erts_do_exit_process(Process* p, Eterm reason) cancel_suspend_of_suspendee(p, ERTS_PROC_LOCKS_ALL); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); -#endif + ERTS_MSGQ_MV_INQ2PRIVQ(p); if (IS_TRACED(p)) { if (IS_TRACED_FL(p, F_TRACE_CALLS)) @@ -13835,7 +12995,7 @@ erts_do_exit_process(Process* p, Eterm reason) ASSERT(erts_proc_read_refc(p) > 0); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); if (IS_TRACED_FL(p,F_TRACE_PROCS)) trace_proc(p, ERTS_PROC_LOCK_MAIN, p, am_exit, reason); @@ -13857,7 +13017,7 @@ erts_continue_exit_process(Process *p) ErtsMonitor *mon; ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; Eterm reason = p->fvalue; - DistEntry *dep; + DistEntry *dep = NULL; erts_aint32_t state; int delay_del_proc = 0; @@ -13865,7 +13025,7 @@ erts_continue_exit_process(Process *p) int yield_allowed = 1; #endif - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); ASSERT(ERTS_PROC_IS_EXITING(p)); @@ -13879,7 +13039,6 @@ erts_continue_exit_process(Process *p) p->bif_timers = NULL; } -#ifdef ERTS_SMP if (p->flags & F_SCHDLR_ONLN_WAITQ) abort_sched_onln_chng_waitq(p); @@ -13923,7 +13082,6 @@ erts_continue_exit_process(Process *p) __FILE__, __LINE__, (int) ssr); } } -#endif if (p->flags & F_USING_DB) { if (erts_db_process_exiting(p, ERTS_PROC_LOCK_MAIN)) @@ -13932,24 +13090,20 @@ erts_continue_exit_process(Process *p) } erts_set_gc_state(p, 1); - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); if (state & ERTS_PSFLG_ACTIVE_SYS -#ifdef ERTS_DIRTY_SCHEDULERS || p->dirty_sys_tasks -#endif ) { if (cleanup_sys_tasks(p, state, CONTEXT_REDS) >= CONTEXT_REDS/2) goto yield; } #ifdef DEBUG - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); ASSERT(p->sys_task_qs == NULL); ASSERT(ERTS_PROC_GET_DELAYED_GC_TASK_QS(p) == NULL); -#ifdef ERTS_DIRTY_SCHEDULERS ASSERT(p->dirty_sys_tasks == NULL); -#endif - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); #endif if (p->flags & F_USING_DDLL) { @@ -13982,7 +13136,7 @@ erts_continue_exit_process(Process *p) if (IS_TRACED_FL(p, F_TRACE_SCHED_EXIT)) trace_sched(p, curr_locks, am_out_exited); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); curr_locks = ERTS_PROC_LOCKS_ALL; /* @@ -14005,23 +13159,19 @@ erts_continue_exit_process(Process *p) ErtsRunQueue *rq; rq = erts_get_runq_current(erts_proc_sched_data(p)); - erts_smp_runq_lock(rq); + erts_runq_lock(rq); -#ifdef ERTS_SMP ASSERT(p->scheduler_data); ASSERT(p->scheduler_data->current_process == p); ASSERT(p->scheduler_data->free_process == NULL); p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; -#else - erts_proc_inc_refc(p); /* Decremented in schedule() */ -#endif /* Time of death! */ erts_ptab_delete_element(&erts_proc, &p->common); - erts_smp_runq_unlock(rq); + erts_runq_unlock(rq); } /* @@ -14033,7 +13183,7 @@ erts_continue_exit_process(Process *p) { /* Inactivate and notify free */ - erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t n, e, a = erts_atomic32_read_nob(&p->state); int refc_inced = 0; while (1) { n = e = a; @@ -14044,12 +13194,11 @@ erts_continue_exit_process(Process *p) erts_proc_inc_refc(p); refc_inced = 1; } - a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } -#ifdef ERTS_DIRTY_SCHEDULERS if (a & (ERTS_PSFLG_DIRTY_RUNNING | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { p->flags |= F_DELAYED_DEL_PROC; @@ -14059,18 +13208,20 @@ erts_continue_exit_process(Process *p) * when done with the process... */ } -#endif if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) erts_proc_dec_refc(p); } - - dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + dep = ((p->flags & F_DISTRIBUTION) + ? ERTS_PROC_SET_DIST_ENTRY(p, NULL) + : NULL); + + erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL); if (dep) { - erts_do_net_exits(dep, reason); + erts_do_net_exits(dep, (reason == am_kill) ? am_killed : reason); + erts_deref_dist_entry(dep); } /* @@ -14102,20 +13253,16 @@ erts_continue_exit_process(Process *p) have none here */ } -#ifdef ERTS_SMP - erts_flush_trace_messages(p, 0); -#endif + erts_proc_lock(p, ERTS_PROC_LOCK_MAIN); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + + erts_flush_trace_messages(p, ERTS_PROC_LOCK_MAIN); ERTS_TRACER_CLEAR(&ERTS_TRACER(p)); if (!delay_del_proc) delete_process(p); -#ifdef ERTS_SMP - erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); -#endif - return; yield: @@ -14124,20 +13271,20 @@ erts_continue_exit_process(Process *p) ASSERT(yield_allowed); #endif - ERTS_SMP_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); + ERTS_LC_ASSERT(curr_locks == erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & curr_locks); p->i = (BeamInstr *) beam_continue_exit; if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) { - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); curr_locks |= ERTS_PROC_LOCK_STATUS; } if (curr_locks != ERTS_PROC_LOCK_MAIN) - erts_smp_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks); + erts_proc_unlock(p, ~ERTS_PROC_LOCK_MAIN & curr_locks); - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); BUMP_ALL_REDS(p); } @@ -14173,7 +13320,7 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "CP: %p (", p->cp); print_function_from_pc(to, to_arg, p->cp); erts_print(to, to_arg, ")\n"); - state = erts_smp_atomic32_read_acqb(&p->state); + state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_GC))) { @@ -14254,8 +13401,7 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); -#ifdef ERTS_SMP - flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags); + flg = erts_atomic32_read_dirty(&esdp->ssi->flags); erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -14282,7 +13428,6 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); -#endif flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work); erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: "); @@ -14325,12 +13470,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { break; } erts_print(to, to_arg, "Length: %d\n", - erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + erts_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); } erts_print(to, to_arg, "Run Queue Port Length: %d\n", - erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + erts_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); - flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags); + flg = erts_atomic32_read_dirty(&esdp->run_queue->flags); erts_print(to, to_arg, "Run Queue Flags: "); for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { erts_aint32_t chk = (1 << i); @@ -14402,7 +13547,7 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { p = esdp->current_process; erts_print(to, to_arg, "Current Process: "); if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { - flg = erts_smp_atomic32_read_dirty(&p->state); + flg = erts_atomic32_read_dirty(&p->state); erts_print(to, to_arg, "%T\n", p->common.id); erts_print(to, to_arg, "Current Process State: "); @@ -14452,19 +13597,17 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { */ void erts_halt(int code) { - if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, + if (-1 == erts_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, -1)) { -#ifdef ERTS_DIRTY_SCHEDULERS ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); -#endif erts_halt_code = code; notify_reap_ports_relb(); } } -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p) { @@ -14484,3 +13627,24 @@ erts_dbg_check_halloc_lock(Process *p) return 0; } #endif + +void +erts_debug_later_op_foreach(void (*callback)(void*), + void (*func)(void *, ErtsThrPrgrVal, void *), + void *arg) +{ + int six; + if (!erts_thr_progress_is_blocking()) + ERTS_INTERNAL_ERROR("Not blocking thread progress"); + + for (six = 0; six < erts_no_schedulers; six++) { + ErtsSchedulerData *esdp = &erts_aligned_scheduler_data[six].esd; + ErtsThrPrgrLaterOp *lop = esdp->aux_work_data.later_op.first; + + while (lop) { + if (lop->func == callback) + func(arg, lop->later, lop->data); + lop = lop->next; + } + } +} diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9d7ba27c50..66d7848f89 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -47,7 +47,6 @@ typedef struct process Process; #include "erl_port.h" #undef ERL_PORT_GET_PORT_TYPE_ONLY__ #include "erl_vm.h" -#include "erl_smp.h" #include "erl_message.h" #include "erl_process_dict.h" #include "erl_node_container_utils.h" @@ -106,27 +105,20 @@ struct saved_calls { }; extern Export exp_send, exp_receive, exp_timeout; -extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; extern Uint erts_no_total_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS extern Uint erts_no_dirty_cpu_schedulers; extern Uint erts_no_dirty_io_schedulers; -#endif extern Uint erts_no_run_queues; extern int erts_sched_thread_suggested_stack_size; -#ifdef ERTS_DIRTY_SCHEDULERS extern int erts_dcpu_sched_thread_suggested_stack_size; extern int erts_dio_sched_thread_suggested_stack_size; -#endif #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20 /* Kilo words */ #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ -#ifdef ERTS_SMP #include "erl_bits.h" -#endif /* process priorities */ #define PRIORITY_MAX 0 @@ -224,31 +216,31 @@ extern int erts_dio_sched_thread_suggested_stack_size; ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) #define ERTS_RUNQ_FLGS_INIT(RQ, INIT) \ - erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT)) + erts_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT)) #define ERTS_RUNQ_FLGS_SET(RQ, FLGS) \ - ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_bor_relb(&(RQ)->flags, \ (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS) \ - ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_bor_nob(&(RQ)->flags, \ (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS) \ - ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, \ (erts_aint32_t) (MSK), \ (erts_aint32_t) (FLGS))) #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS) \ - ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_band_relb(&(RQ)->flags, \ (erts_aint32_t) ~(FLGS))) #define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS) \ - ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_band_nob(&(RQ)->flags, \ (erts_aint32_t) ~(FLGS))) #define ERTS_RUNQ_FLGS_GET(RQ) \ - ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags)) + ((Uint32) erts_atomic32_read_acqb(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_NOB(RQ) \ - ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags)) + ((Uint32) erts_atomic32_read_nob(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_GET_MB(RQ) \ - ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags)) + ((Uint32) erts_atomic32_read_mb(&(RQ)->flags)) #define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS) \ - ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, \ + ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, \ (erts_aint32_t) (MSK), \ (erts_aint32_t) (FLGS))) @@ -365,20 +357,18 @@ typedef enum { typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; -#ifdef ERTS_DIRTY_SCHEDULERS typedef struct { - erts_smp_spinlock_t lock; + erts_spinlock_t lock; ErtsSchedulerSleepInfo *list; } ErtsSchedulerSleepList; -#endif struct ErtsSchedulerSleepInfo_ { -#ifdef ERTS_SMP + struct ErtsSchedulerData_ *esdp; ErtsSchedulerSleepInfo *next; ErtsSchedulerSleepInfo *prev; - erts_smp_atomic32_t flags; + erts_atomic32_t flags; erts_tse_t *event; -#endif + struct erts_poll_thread *psi; erts_atomic32_t aux_work; }; @@ -422,7 +412,7 @@ typedef struct ErtsSchedulerData_ ErtsSchedulerData; typedef struct ErtsRunQueue_ ErtsRunQueue; typedef struct { - erts_smp_atomic32_t len; + erts_atomic32_t len; erts_aint32_t max_len; int reds; } ErtsRunQueueInfo; @@ -433,7 +423,6 @@ typedef struct { # define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1 #endif -#ifdef ERTS_SMP #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT @@ -476,30 +465,25 @@ struct ErtsMigrationPaths_ { ErtsMigrationPath mpath[1]; }; -#endif /* ERTS_SMP */ struct ErtsRunQueue_ { int ix; - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; + erts_mtx_t mtx; + erts_cnd_t cnd; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP ErtsSchedulerSleepList sleepers; -#endif -#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ int woken; - erts_smp_atomic32_t flags; + erts_atomic32_t flags; int check_balance_reds; int full_reds_history_sum; int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE]; int out_of_work_count; erts_aint32_t max_len; - erts_smp_atomic32_t len; + erts_atomic32_t len; int wakeup_other; int wakeup_other_reds; @@ -518,7 +502,7 @@ struct ErtsRunQueue_ { struct { ErtsMiscOpList *start; ErtsMiscOpList *end; - erts_smp_atomic_t evac_runq; + erts_atomic_t evac_runq; } misc; struct { @@ -531,9 +515,7 @@ struct ErtsRunQueue_ { #endif }; -#ifdef ERTS_SMP extern long erts_runq_supervision_interval; -#endif typedef union { ErtsRunQueue runq; @@ -581,17 +563,12 @@ typedef struct { int sched_id; ErtsSchedulerData *esdp; ErtsSchedulerSleepInfo *ssi; -#ifdef ERTS_SMP ErtsThrPrgrVal current_thr_prgr; ErtsThrPrgrVal latest_wakeup; -#endif struct { int ix; -#ifdef ERTS_SMP ErtsThrPrgrVal thr_prgr; -#endif } misc; -#ifdef ERTS_SMP struct { ErtsThrPrgrVal thr_prgr; } dd; @@ -604,24 +581,17 @@ typedef struct { ErtsThrPrgrLaterOp *first; ErtsThrPrgrLaterOp *last; } later_op; -#endif -#ifdef ERTS_USE_ASYNC_READY_Q struct { -#ifdef ERTS_SMP int need_thr_prgr; ErtsThrPrgrVal thr_prgr; -#endif void *queue; } async_ready; -#endif -#ifdef ERTS_SMP struct { Uint64 next; int *sched2jix; int jix; ErtsDelayedAuxWorkWakeupJob *job; } delayed_wakeup; -#endif struct { ErtsEtsAllYieldData ets_all; /* Other yielding operations... */ @@ -639,13 +609,11 @@ typedef struct { (&(ESDP)->aux_work_data.yield.NAME) void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp); -#ifdef ERTS_DIRTY_SCHEDULERS typedef enum { ERTS_DIRTY_CPU_SCHEDULER, ERTS_DIRTY_IO_SCHEDULER } ErtsDirtySchedulerType; -#endif struct ErtsSchedulerData_ { /* @@ -659,21 +627,17 @@ struct ErtsSchedulerData_ { ErtsTimerWheel *timer_wheel; ErtsNextTimeoutRef next_tmo_ref; ErtsHLTimerService *timer_service; -#ifdef ERTS_SMP ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ Process *free_process; ErtsThrPrgrData thr_progress_data; -#endif ErtsSchedulerSleepInfo *ssi; Process *current_process; ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ -#ifdef ERTS_DIRTY_SCHEDULERS Uint dirty_no; /* Scheduler number for dirty schedulers */ Process *dirty_shadow_process; -#endif Port *current_port; ErtsRunQueue *run_queue; int virtual_reds; @@ -712,25 +676,23 @@ typedef union { } ErtsAlignedSchedulerData; extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -#ifdef ERTS_DIRTY_SCHEDULERS extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; -#endif -#ifndef ERTS_SMP -extern ErtsSchedulerData *erts_scheduler_data; -#endif -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) -int erts_smp_lc_runq_is_locked(ErtsRunQueue *); +#if defined(ERTS_ENABLE_LOCK_CHECK) +int erts_lc_runq_is_locked(ErtsRunQueue *); #endif +void +erts_debug_later_op_foreach(void (*callback)(void*), + void (*func)(void *, ErtsThrPrgrVal, void *), + void *arg); + #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -#ifdef ERTS_SMP void erts_empty_runq(ErtsRunQueue *rq); void erts_non_empty_runq(ErtsRunQueue *rq); -#endif /* @@ -738,86 +700,84 @@ void erts_non_empty_runq(ErtsRunQueue *rq); * other threads peek at values without run queue lock. */ -ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); -ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); -ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi); +ERTS_GLB_INLINE void erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio); +ERTS_GLB_INLINE void erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) { erts_aint32_t len; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_dirty(&rq->len); + len = erts_atomic32_read_dirty(&rq->len); -#ifdef ERTS_SMP if (len == 0) erts_non_empty_runq(rq); -#endif len++; if (rq->max_len < len) rq->max_len = len; ASSERT(len > 0); - erts_smp_atomic32_set_nob(&rq->len, len); + erts_atomic32_set_nob(&rq->len, len); - len = erts_smp_atomic32_read_dirty(&rqi->len); + len = erts_atomic32_read_dirty(&rqi->len); ASSERT(len >= 0); if (len == 0) { - ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + ASSERT((erts_atomic32_read_nob(&rq->flags) & ((erts_aint32_t) (1 << prio))) == 0); - erts_smp_atomic32_read_bor_nob(&rq->flags, + erts_atomic32_read_bor_nob(&rq->flags, (erts_aint32_t) (1 << prio)); } len++; if (rqi->max_len < len) rqi->max_len = len; - erts_smp_atomic32_set_relb(&rqi->len, len); + erts_atomic32_set_relb(&rqi->len, len); } ERTS_GLB_INLINE void -erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) +erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio) { erts_aint32_t len; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_dirty(&rq->len); + len = erts_atomic32_read_dirty(&rq->len); len--; ASSERT(len >= 0); - erts_smp_atomic32_set_nob(&rq->len, len); + erts_atomic32_set_nob(&rq->len, len); - len = erts_smp_atomic32_read_dirty(&rqi->len); + len = erts_atomic32_read_dirty(&rqi->len); len--; ASSERT(len >= 0); if (len == 0) { - ASSERT((erts_smp_atomic32_read_nob(&rq->flags) + ASSERT((erts_atomic32_read_nob(&rq->flags) & ((erts_aint32_t) (1 << prio)))); - erts_smp_atomic32_read_band_nob(&rq->flags, + erts_atomic32_read_band_nob(&rq->flags, ~((erts_aint32_t) (1 << prio))); } - erts_smp_atomic32_set_relb(&rqi->len, len); + erts_atomic32_set_relb(&rqi->len, len); } ERTS_GLB_INLINE void -erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) +erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) { erts_aint32_t len; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq)); - len = erts_smp_atomic32_read_dirty(&rqi->len); + len = erts_atomic32_read_dirty(&rqi->len); ASSERT(rqi->max_len >= len); rqi->max_len = len; } #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X)) +#define RUNQ_READ_LEN(X) erts_atomic32_read_nob((X)) #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ @@ -835,14 +795,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_NIF_TRAP_EXPORT 5 #define ERTS_PSD_ETS_OWNED_TABLES 6 #define ERTS_PSD_ETS_FIXED_TABLES 7 -#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 8 +#define ERTS_PSD_DIST_ENTRY 8 +#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 9 /* keep last... */ -#define ERTS_PSD_SIZE 9 +#define ERTS_PSD_SIZE 10 #if !defined(HIPE) # undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF # undef ERTS_PSD_SIZE -# define ERTS_PSD_SIZE 8 +# define ERTS_PSD_SIZE 9 #endif typedef struct { @@ -876,6 +837,9 @@ typedef struct { #define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -890,7 +854,7 @@ extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #define ERTS_SCHED_STAT_MODIFY_CLEAR 3 typedef struct { - erts_smp_spinlock_t lock; + erts_spinlock_t lock; int enabled; struct { Eterm name; @@ -911,7 +875,6 @@ typedef struct { typedef struct ErtsProcSysTask_ ErtsProcSysTask; typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs; -#ifdef ERTS_SMP typedef struct ErtsPendingSuspend_ ErtsPendingSuspend; struct ErtsPendingSuspend_ { @@ -924,7 +887,6 @@ struct ErtsPendingSuspend_ { Eterm pid); }; -#endif /* Defines to ease the change of memory architecture */ @@ -1076,23 +1038,18 @@ struct process { ErlHeapFragment* live_hf_end; ErtsMessage *msg_frag; /* Pointer to message fragment list */ Uint mbuf_sz; /* Total size of heap fragments and message fragments */ - erts_smp_atomic_t psd; /* Rarely used process specific data */ + erts_atomic_t psd; /* Rarely used process specific data */ Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; -#ifdef ERTS_DIRTY_SCHEDULERS ErtsProcSysTask *dirty_sys_tasks; -#endif - erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ -#endif + erts_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ + erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */ -#ifdef ERTS_SMP ErlMessageInQueue msg_inq; ErlTraceMessageQueue *trace_msg_q; ErtsPendExit pending_exit; @@ -1100,11 +1057,10 @@ struct process { ErtsSchedulerData *scheduler_data; Eterm suspendee; ErtsPendingSuspend *pending_suspenders; - erts_smp_atomic_t run_queue; + erts_atomic_t run_queue; #ifdef HIPE struct hipe_process_state_smp hipe_smp; #endif -#endif #ifdef CHECK_FOR_HOLES Eterm* last_htop; /* No need to scan the heap below this point. */ @@ -1249,7 +1205,6 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) -#ifdef ERTS_DIRTY_SCHEDULERS /* * Flags in the dirty_state field. @@ -1276,7 +1231,6 @@ void erts_check_for_holes(Process* p); | ERTS_PDSFLG_IN_CPU_PRQ_HIGH \ | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\ | ERTS_PDSFLG_IN_CPU_PRQ_LOW) -#endif /* @@ -1367,7 +1321,7 @@ Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif -extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; +extern erts_rwmtx_t erts_cpu_bind_rwmtx; /* If any of the erts_system_monitor_* variables are set (enabled), ** erts_system_monitor must be != NIL, to allow testing on just ** the erts_system_monitor_* variables. @@ -1541,20 +1495,14 @@ extern int erts_system_profile_ts_type; } \ } while (0) -#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) #define ERTS_NUM_DIRTY_CPU_RUNQS 1 #define ERTS_NUM_DIRTY_IO_RUNQS 1 -#else -#define ERTS_NUM_DIRTY_CPU_RUNQS 0 -#define ERTS_NUM_DIRTY_IO_RUNQS 0 -#endif #define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS) #define ERTS_RUNQ_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ &erts_aligned_run_queues[(IX)].runq) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_RUNQ_IX_IS_DIRTY(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \ (erts_no_run_queues <= (IX))) @@ -1565,13 +1513,9 @@ extern int erts_system_profile_ts_type; #define ERTS_DIRTY_IO_RUNQ (&erts_aligned_run_queues[erts_no_run_queues+1].runq) #define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ) #define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ) -#else -#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 -#endif #define ERTS_SCHEDULER_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers), \ &erts_aligned_scheduler_data[(IX)].esd) -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_DIRTY_CPU_SCHEDULER_IX(IX) \ (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers), \ &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd) @@ -1584,24 +1528,12 @@ extern int erts_system_profile_ts_type; ((ESDP)->type == ERTS_SCHED_DIRTY_CPU) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ ((ESDP)->type == ERTS_SCHED_DIRTY_IO) -#else /* !ERTS_DIRTY_SCHEDULERS */ -#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0 -#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 -#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0 -#endif void erts_pre_init_process(void); void erts_late_init_process(void); void erts_early_init_scheduling(int); -void erts_init_scheduling(int, int -#ifdef ERTS_DIRTY_SCHEDULERS - , int, int, int -#endif - ); -#ifdef ERTS_DIRTY_SCHEDULERS +void erts_init_scheduling(int, int, int, int, int, int); void erts_execute_dirty_system_task(Process *c_p); -#endif int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable, int dirty_cpu, int want_dirty_io); @@ -1612,6 +1544,7 @@ Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); ErtsProcList *erts_proclist_create(Process *); +ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); @@ -1798,14 +1731,11 @@ void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*); void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); #endif -#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS) void erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *); -#endif -#ifdef ERTS_SMP ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, @@ -1820,14 +1750,9 @@ void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); void erts_alloc_ensure_handle_delayed_dealloc_call(int); void erts_notify_canceled_timer(ErtsSchedulerData *, int); -#endif -#if ERTS_USE_ASYNC_READY_Q void erts_notify_check_async_ready_queue(void *); -#endif -#ifdef ERTS_SMP void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later); void erts_notify_finish_breakpointing(Process* p); -#endif void erts_schedule_misc_aux_work(int sched_id, void (*func)(void *), void *arg); @@ -1896,13 +1821,9 @@ int erts_send_exit_signal(Process *, Eterm, Process *, Uint32); -#ifdef ERTS_SMP void erts_handle_pending_exit(Process *, ErtsProcLocks); #define ERTS_PROC_PENDING_EXIT(P) \ - (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state)) -#else -#define ERTS_PROC_PENDING_EXIT(P) 0 -#endif + (ERTS_PSFLG_PENDING_EXIT & erts_atomic32_read_acqb(&(P)->state)) void erts_deep_process_dump(fmtfn_t, void *); @@ -1932,19 +1853,7 @@ do { \ # define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP) #endif -#if defined(ERTS_SMP) || defined(USE_THREADS) ErtsSchedulerData *erts_get_scheduler_data(void); -#else -ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE -ErtsSchedulerData *erts_get_scheduler_data(void) -{ - return erts_scheduler_data; -} -#endif -#endif void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); @@ -1956,7 +1865,7 @@ ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) { /* No barrier needed, due to msg lock */ - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = erts_atomic32_read_nob(&p->state); if (!(state & ERTS_PSFLG_ACTIVE)) erts_schedule_process(p, state, locks); } @@ -1966,7 +1875,7 @@ erts_schedule_dirty_sys_execution(Process *c_p) { erts_aint32_t a, n, e; - a = erts_smp_atomic32_read_nob(&c_p->state); + a = erts_atomic32_read_nob(&c_p->state); /* * Only a currently executing process schedules @@ -1982,7 +1891,7 @@ erts_schedule_dirty_sys_execution(Process *c_p) | ERTS_PSFLG_PENDING_EXIT))) { e = a; n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS; - a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e); + a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e); if (a == e) break; /* dirty-active-sys set */ } @@ -1990,21 +1899,21 @@ erts_schedule_dirty_sys_execution(Process *c_p) #endif -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) #define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ #include "erl_process_lock.h" #undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__ -#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) \ +#define ERTS_LC_CHK_RUNQ_LOCK(RQ, L) \ do { \ if ((L)) \ - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ))); \ + ERTS_LC_ASSERT(erts_lc_runq_is_locked((RQ))); \ else \ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ))); \ + ERTS_LC_ASSERT(!erts_lc_runq_is_locked((RQ))); \ } while (0) #else -#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L) +#define ERTS_LC_CHK_RUNQ_LOCK(RQ, L) #endif void *erts_psd_set_init(Process *p, int ix, void *data); @@ -2020,22 +1929,22 @@ ERTS_GLB_INLINE void * erts_psd_get(Process *p, int ix) { ErtsPSD *psd; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks) - ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].get_locks; - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks + ERTS_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks || erts_thr_progress_is_blocking()); } #endif - psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); if (!psd) return NULL; - ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; return psd->data[ix]; } @@ -2043,30 +1952,28 @@ ERTS_GLB_INLINE void * erts_psd_set(Process *p, int ix, void *data) { ErtsPSD *psd; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); - erts_aint32_t state = state = erts_smp_atomic32_read_nob(&p->state); + erts_aint32_t state = state = erts_atomic32_read_nob(&p->state); if (!(state & ERTS_PSFLG_FREE)) { if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) - ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking()); else { locks &= erts_psd_required_locks[ix].set_locks; - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks + ERTS_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks || erts_thr_progress_is_blocking()); } } #endif - psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd); ASSERT(0 <= ix && ix < ERTS_PSD_SIZE); if (psd) { void *old; -#ifdef ERTS_SMP #ifdef ETHR_ORDERED_READ_DEPEND ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); #else ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore); #endif -#endif old = psd->data[ix]; psd->data[ix] = data; return old; @@ -2103,6 +2010,11 @@ erts_psd_set(Process *p, int ix, void *data) #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#define ERTS_PROC_GET_DIST_ENTRY(P) \ + ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) +#define ERTS_PROC_SET_DIST_ENTRY(P, DE) \ + ((DistEntry *) erts_psd_set((P), ERTS_PSD_DIST_ENTRY, (void *) (DE))) + #ifdef HIPE #define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \ ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF)) @@ -2146,7 +2058,6 @@ erts_proc_set_error_handler(Process *p, Eterm handler) #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -#ifdef ERTS_SMP #include "erl_thr_progress.h" @@ -2248,7 +2159,6 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) #endif -#endif #endif @@ -2259,13 +2169,13 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void); ERTS_GLB_INLINE Uint erts_get_scheduler_id(void); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p); ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp); -ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq); -ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq); -ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq); -ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq); -ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); -ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); -ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); +ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq); +ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq); +ERTS_GLB_INLINE void erts_runq_unlock(ErtsRunQueue *rq); +ERTS_GLB_INLINE void erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq); +ERTS_GLB_INLINE void erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq); +ERTS_GLB_INLINE void erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); +ERTS_GLB_INLINE void erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2); ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp, erts_aint32_t *psp, @@ -2290,11 +2200,7 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p) { ErtsSchedulerData *esdp; ASSERT(c_p); -#if !defined(ERTS_SMP) - esdp = erts_get_scheduler_data(); -#else esdp = c_p->scheduler_data; -# if defined(ERTS_DIRTY_SCHEDULERS) if (esdp) { ASSERT(esdp == erts_get_scheduler_data()); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); @@ -2304,8 +2210,6 @@ ErtsSchedulerData *erts_proc_sched_data(Process *c_p) ASSERT(esdp); ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); } -# endif -#endif ASSERT(esdp); return esdp; } @@ -2336,124 +2240,94 @@ Eterm erts_get_current_pid(void) ERTS_GLB_INLINE Uint erts_get_scheduler_id(void) { -#ifdef ERTS_SMP ErtsSchedulerData *esdp = erts_get_scheduler_data(); -#ifdef ERTS_DIRTY_SCHEDULERS if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)) return 0; else -#endif return esdp ? esdp->no : (Uint) 0; -#else - return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0; -#endif } ERTS_GLB_INLINE ErtsRunQueue * erts_get_runq_proc(Process *p) { -#ifdef ERTS_SMP ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue)); return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue); -#else - return ERTS_RUNQ_IX(0); -#endif } ERTS_GLB_INLINE ErtsRunQueue * erts_get_runq_current(ErtsSchedulerData *esdp) { ASSERT(!esdp || esdp == erts_get_scheduler_data()); -#ifdef ERTS_SMP if (!esdp) esdp = erts_get_scheduler_data(); return esdp->run_queue; -#else - return ERTS_RUNQ_IX(0); -#endif } ERTS_GLB_INLINE void -erts_smp_runq_lock(ErtsRunQueue *rq) +erts_runq_lock(ErtsRunQueue *rq) { -#ifdef ERTS_SMP - erts_smp_mtx_lock(&rq->mtx); -#endif + erts_mtx_lock(&rq->mtx); } ERTS_GLB_INLINE int -erts_smp_runq_trylock(ErtsRunQueue *rq) +erts_runq_trylock(ErtsRunQueue *rq) { -#ifdef ERTS_SMP - return erts_smp_mtx_trylock(&rq->mtx); -#else - return 0; -#endif + return erts_mtx_trylock(&rq->mtx); } ERTS_GLB_INLINE void -erts_smp_runq_unlock(ErtsRunQueue *rq) +erts_runq_unlock(ErtsRunQueue *rq) { -#ifdef ERTS_SMP - erts_smp_mtx_unlock(&rq->mtx); -#endif + erts_mtx_unlock(&rq->mtx); } ERTS_GLB_INLINE void -erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) +erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx)); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&rq->mtx)); if (xrq != rq) { - if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) { + if (erts_mtx_trylock(&xrq->mtx) == EBUSY) { if (rq < xrq) - erts_smp_mtx_lock(&xrq->mtx); + erts_mtx_lock(&xrq->mtx); else { - erts_smp_mtx_unlock(&rq->mtx); - erts_smp_mtx_lock(&xrq->mtx); - erts_smp_mtx_lock(&rq->mtx); + erts_mtx_unlock(&rq->mtx); + erts_mtx_lock(&xrq->mtx); + erts_mtx_lock(&rq->mtx); } } } -#endif } ERTS_GLB_INLINE void -erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq) +erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq) { -#ifdef ERTS_SMP if (xrq != rq) - erts_smp_mtx_unlock(&xrq->mtx); -#endif + erts_mtx_unlock(&xrq->mtx); } ERTS_GLB_INLINE void -erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) +erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) { -#ifdef ERTS_SMP ASSERT(rq1 && rq2); if (rq1 == rq2) - erts_smp_mtx_lock(&rq1->mtx); + erts_mtx_lock(&rq1->mtx); else if (rq1 < rq2) { - erts_smp_mtx_lock(&rq1->mtx); - erts_smp_mtx_lock(&rq2->mtx); + erts_mtx_lock(&rq1->mtx); + erts_mtx_lock(&rq2->mtx); } else { - erts_smp_mtx_lock(&rq2->mtx); - erts_smp_mtx_lock(&rq1->mtx); + erts_mtx_lock(&rq2->mtx); + erts_mtx_lock(&rq1->mtx); } -#endif } ERTS_GLB_INLINE void -erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) +erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2) { -#ifdef ERTS_SMP ASSERT(rq1 && rq2); - erts_smp_mtx_unlock(&rq1->mtx); + erts_mtx_unlock(&rq1->mtx); if (rq1 != rq2) - erts_smp_mtx_unlock(&rq2->mtx); -#endif + erts_mtx_unlock(&rq2->mtx); } ERTS_GLB_INLINE ErtsMessage * @@ -2485,7 +2359,7 @@ erts_alloc_message_heap(Process *pp, Eterm **hpp, ErlOffHeap **ohpp) { - erts_aint32_t state = pp ? erts_smp_atomic32_read_nob(&pp->state) : 0; + erts_aint32_t state = pp ? erts_atomic32_read_nob(&pp->state) : 0; return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp); } @@ -2499,7 +2373,7 @@ erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, *msgpp = erts_shrink_message(*msgpp, used_hp - start_hp, brefs, brefs_size); else if (!(*msgpp)->data.attached) { - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN + ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(pp)); HRelease(pp, end_hp, used_hp); } @@ -2561,7 +2435,6 @@ ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end) } #endif -#ifdef ERTS_SMP Process *erts_pid2proc_not_running(Process *, ErtsProcLocks, @@ -2574,37 +2447,29 @@ Process *erts_pid2proc_nropt(Process *c_p, extern int erts_disable_proc_not_running_opt; #ifdef DEBUG -#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \ +#define ERTS_ASSERT_IS_NOT_EXITING(P) \ do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0) #else -#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) +#define ERTS_ASSERT_IS_NOT_EXITING(P) #endif -#else /* !ERTS_SMP */ - -#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) - -#define erts_pid2proc_not_running erts_pid2proc -#define erts_pid2proc_nropt erts_pid2proc - -#endif #define ERTS_PROC_IS_EXITING(P) \ - (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state)) + (ERTS_PSFLG_EXITING & erts_atomic32_read_acqb(&(P)->state)) /* Minimum NUMBER of processes for a small system to start */ #define ERTS_MIN_PROCESSES 1024 -#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS +#if ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS #undef ERTS_MIN_PROCESSES #define ERTS_MIN_PROCESSES ERTS_NO_OF_PIX_LOCKS #endif -void erts_smp_notify_inc_runq(ErtsRunQueue *runq); +void erts_notify_inc_runq(ErtsRunQueue *runq); -#ifdef ERTS_SMP void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); +void erts_aux_thread_poke(void); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -2613,9 +2478,9 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t flags; ERTS_THR_MEMORY_BARRIER; - flags = erts_smp_atomic32_read_nob(&ssi->flags); + flags = erts_atomic32_read_nob(&ssi->flags); if (flags & ERTS_SSI_FLG_SLEEPING) { - flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + flags = erts_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); erts_sched_finish_poke(ssi, flags); } } @@ -2623,7 +2488,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* #ifdef ERTS_SMP */ #include "erl_process_lock.h" @@ -2633,5 +2497,5 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) void erts_halt(int code); -extern erts_smp_atomic32_t erts_halt_progress; +extern erts_atomic32_t erts_halt_progress; extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index b826e6c5d3..5a2c262ff1 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -69,7 +69,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); + erts_aint32_t state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_EXITING|ERTS_PSFLG_GC))) dump_process_info(to, to_arg, p); } @@ -85,7 +85,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { size += sizeof(Process); if (incl_msg_inq) - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + ERTS_MSGQ_MV_INQ2PRIVQ(p); erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); @@ -106,7 +106,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { size += p->arity * sizeof(p->arg_reg[0]); } - if (erts_smp_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) + if (erts_atomic_read_nob(&p->psd) != (erts_aint_t) NULL) size += sizeof(ErtsPSD); scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); @@ -126,7 +126,7 @@ dump_process_info(fmtfn_t to, void *to_arg, Process *p) ErtsMessage* mp; int yreg = -1; - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + ERTS_MSGQ_MV_INQ2PRIVQ(p); if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0 && p->msg.first) { erts_print(to, to_arg, "=proc_messages:%T\n", p->common.id); diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index c0e7380ed0..431867f27e 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -56,9 +56,9 @@ * Note that wait flags may be read without the pix lock, but * it is important that wait flags only are modified when the pix * lock is held. - * This implementation assumes that erts_smp_atomic_or_retold() + * This implementation assumes that erts_atomic_or_retold() * provides necessary memorybarriers for a lock operation, and that - * erts_smp_atomic_and_retold() provides necessary memorybarriers + * erts_atomic_and_retold() provides necessary memorybarriers * for an unlock operation. */ @@ -69,7 +69,6 @@ #include "erl_process.h" #include "erl_thr_progress.h" -#ifdef ERTS_SMP #if ERTS_PROC_LOCK_OWN_IMPL @@ -112,21 +111,13 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -#ifdef ERTS_ENABLE_LOCK_COUNT -static void lcnt_enable_proc_lock_count(Process *proc, int enable); -#endif - void erts_init_proc_lock(int cpus) { int i; for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_mtx_init_x(&erts_pix_locks[i].u.mtx, - "pix_lock", make_small(i)); -#else - erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif + erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_PROCESS); } #if ERTS_PROC_LOCK_OWN_IMPL erts_thr_install_exit_handler(cleanup_tse); @@ -472,7 +463,7 @@ wait_for_locks(Process *p, } /* - * erts_proc_lock_failed() is called when erts_smp_proc_lock() + * erts_proc_lock_failed() is called when erts_proc_lock() * wasn't able to lock all locks. We may need to transfer locks * to waiters and wait for our turn on locks. * @@ -551,7 +542,7 @@ erts_proc_lock_failed(Process *p, } /* - * erts_proc_unlock_failed() is called when erts_smp_proc_unlock() + * erts_proc_unlock_failed() is called when erts_proc_unlock() * wasn't able to unlock all locks. We may need to transfer locks * to waiters. */ @@ -717,7 +708,7 @@ proc_safelock(int is_managed, refc1 = 1; erts_proc_inc_refc(p1); } - erts_smp_proc_unlock(p1, unlock_locks); + erts_proc_unlock(p1, unlock_locks); } unlock_locks = unlock_mask & have_locks2; if (unlock_locks) { @@ -727,7 +718,7 @@ proc_safelock(int is_managed, refc2 = 1; erts_proc_inc_refc(p2); } - erts_smp_proc_unlock(p2, unlock_locks); + erts_proc_unlock(p2, unlock_locks); } } @@ -758,7 +749,7 @@ proc_safelock(int is_managed, if (need_locks2 & lock) lock_no--; locks = need_locks1 & lock_mask; - erts_smp_proc_lock(p1, locks); + erts_proc_lock(p1, locks); have_locks1 |= locks; need_locks1 &= ~locks; } @@ -769,7 +760,7 @@ proc_safelock(int is_managed, lock = (1 << ++lock_no); } locks = need_locks2 & lock_mask; - erts_smp_proc_lock(p2, locks); + erts_proc_lock(p2, locks); have_locks2 |= locks; need_locks2 &= ~locks; } @@ -906,7 +897,7 @@ erts_pid2proc_opt(Process *c_p, #endif /* ERTS_PROC_LOCK_OWN_IMPL */ { /* Try a quick trylock to grab all the locks we need. */ - busy = (int) erts_smp_proc_raw_trylock__(proc, need_locks); + busy = (int) erts_proc_raw_trylock__(proc, need_locks); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK) erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__); @@ -944,7 +935,7 @@ erts_pid2proc_opt(Process *c_p, erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) - erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); + erts_lcnt_proc_lock_unacquire(&proc->lock, lcnt_locks); #endif managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; @@ -984,7 +975,7 @@ erts_pid2proc_opt(Process *c_p, : (proc != (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) { - erts_smp_proc_unlock(proc, need_locks); + erts_proc_unlock(proc, need_locks); if (flags & ERTS_P2P_FLG_INC_REFC) dec_refc_proc = proc; @@ -1010,11 +1001,9 @@ static ERTS_INLINE Process *proc_lookup_inc_refc(Eterm pid, int allow_exit) { Process *proc; -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl; dhndl = erts_thr_progress_unmanaged_delay(); -#endif proc = erts_proc_lookup_raw(pid); if (proc) { @@ -1024,9 +1013,7 @@ Process *proc_lookup_inc_refc(Eterm pid, int allow_exit) erts_proc_inc_refc(proc); } -#ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); -#endif return proc; } @@ -1050,7 +1037,7 @@ erts_proc_lock_init(Process *p) #if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic32_init_nob(&p->lock.flags, + erts_atomic32_init_nob(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; @@ -1062,32 +1049,38 @@ erts_proc_lock_init(Process *p) #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); + erts_mtx_init(&p->lock.main, "proc_main", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.main.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); + erts_mtx_init(&p->lock.link, "proc_link", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.link.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.link.lc); #endif - erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); + erts_mtx_init(&p->lock.msgq, "proc_msgq", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.msgq.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif - erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id); + erts_mtx_init(&p->lock.btm, "proc_btm", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.btm.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.btm.lc); #endif - erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); + erts_mtx_init(&p->lock.status, "proc_status", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.status.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif - erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id); + erts_mtx_init(&p->lock.trace, "proc_trace", p->common.id, + ERTS_LOCK_FLAGS_CATEGORY_PROCESS); ethr_mutex_lock(&p->lock.trace.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.trace.lc); @@ -1095,7 +1088,7 @@ erts_proc_lock_init(Process *p) #endif #ifdef ERTS_PROC_LOCK_DEBUG for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); + erts_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_init(p); @@ -1115,7 +1108,7 @@ erts_proc_lock_fin(Process *p) erts_mtx_destroy(&p->lock.status); erts_mtx_destroy(&p->lock.trace); #endif -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_COUNT) erts_lcnt_proc_lock_destroy(p); #endif } @@ -1124,117 +1117,70 @@ erts_proc_lock_fin(Process *p) #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) -void erts_lcnt_enable_proc_lock_count(int enable) { - int ix, max = erts_ptab_max(&erts_proc); - Process *proc = NULL; - for (ix = 0; ix < max; ++ix) { - if ((proc = erts_pix2proc(ix)) != NULL) - lcnt_enable_proc_lock_count(proc, enable); - } /* for all processes */ -} - void erts_lcnt_proc_lock_init(Process *p) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { - erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace)); - } else { /* now the common case */ - Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid); - } /* the lock names should really be aligned to four characters */ + erts_lcnt_init_ref(&p->lock.lcnt_carrier); + + if(erts_lcnt_check_enabled(ERTS_LOCK_FLAGS_CATEGORY_PROCESS)) { + erts_lcnt_enable_proc_lock_count(p, 1); + } } /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { - erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_btm)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_trace)); + erts_lcnt_uninstall(&p->lock.lcnt_carrier); } -static void lcnt_enable_proc_lock_count(Process *proc, int enable) { - if (enable) { - if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { - erts_lcnt_proc_lock_init(proc); - } - } - else { - if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { - erts_lcnt_proc_lock_destroy(proc); - } +void erts_lcnt_enable_proc_lock_count(Process *proc, int enable) { + if(proc->common.id == ERTS_INVALID_PID) { + /* Locks without an id are more trouble than they're worth; there's no + * way to look them up and we can't track them with _STATIC since it's + * too early to tell whether we're a system process (proc->static_flags + * hasn't been not set yet). */ + } else if(!enable) { + erts_lcnt_proc_lock_destroy(proc); + } else if(!erts_lcnt_check_ref_installed(&proc->lock.lcnt_carrier)) { + erts_lcnt_lock_info_carrier_t *carrier; + + carrier = erts_lcnt_create_lock_info_carrier(ERTS_LCNT_PROCLOCK_COUNT); + + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, + "proc_main", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, + "proc_link", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, + "proc_msgq", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, + "proc_btm", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, + "proc_status",proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, + "proc_trace", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK); + + erts_lcnt_install(&proc->lock.lcnt_carrier, carrier); } } -void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); } -} +void erts_lcnt_update_process_locks(int enable) { + int i, max; -void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, - char *file, unsigned int line) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); - } - if (locks & ERTS_PROC_LOCK_TRACE) { - erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line); - } -} + max = erts_ptab_max(&erts_proc); -void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); } -} + for(i = 0; i < max; i++) { + int delay_handle; + Process *proc; -void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); } + delay_handle = erts_thr_progress_unmanaged_delay(); + proc = erts_pix2proc(i); + + if(proc != NULL) { + erts_lcnt_enable_proc_lock_count(proc, enable); + } + + if(delay_handle != ERTS_THR_PRGR_DHANDLE_MANAGED) { + erts_thr_progress_unmanaged_continue(delay_handle); + } + } } -void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); } -} /* reversed logic */ + #endif /* ERTS_ENABLE_LOCK_COUNT */ @@ -1249,7 +1195,7 @@ erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_lock_x(&lck,file,line); @@ -1282,7 +1228,7 @@ erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked, { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_trylock_x(locked, &lck, file, line); @@ -1314,7 +1260,7 @@ erts_proc_lc_unlock(Process *p, ErtsProcLocks locks) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_TRACE) { lck.id = lc_id.proc_lock_trace; erts_lc_unlock(&lck); @@ -1349,7 +1295,7 @@ erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_TRACE) { lck.id = lc_id.proc_lock_trace; erts_lc_might_unlock(&lck); @@ -1397,7 +1343,7 @@ erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file, #if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) { lck.id = lc_id.proc_lock_main; erts_lc_require_lock(&lck, file, line); @@ -1444,7 +1390,7 @@ erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks) #if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_TRACE) { lck.id = lc_id.proc_lock_trace; erts_lc_unrequire_lock(&lck); @@ -1493,7 +1439,7 @@ erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks) if (locks & ERTS_PROC_LOCKS_ALL) { erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK); + ERTS_LOCK_TYPE_PROCLOCK); if (locks & ERTS_PROC_LOCK_MAIN) lck.id = lc_id.proc_lock_main; @@ -1524,7 +1470,7 @@ void erts_proc_lc_chk_only_proc_main(Process *p) #if ERTS_PROC_LOCK_OWN_IMPL #define ERTS_PROC_LC_EMPTY_LOCK_INIT \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_PROCLOCK) + ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LOCK_TYPE_PROCLOCK) #endif /* ERTS_PROC_LOCK_OWN_IMPL */ void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks) @@ -1739,22 +1685,22 @@ erts_proc_lc_my_proc_locks(Process *p) #if ERTS_PROC_LOCK_OWN_IMPL erts_lc_lock_t locks[6] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_link, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_status, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK), + ERTS_LOCK_TYPE_PROCLOCK), ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace, p->common.id, - ERTS_LC_FLG_LT_PROCLOCK)}; + ERTS_LOCK_TYPE_PROCLOCK)}; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_lc_lock_t locks[6] = {p->lock.main.lc, p->lock.link.lc, @@ -1834,4 +1780,3 @@ check_queue(erts_proc_lock_t *lck) } #endif -#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 6e704b185d..9d5691d3c4 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -36,7 +36,7 @@ #include "erl_lock_count.h" #endif -#include "erl_smp.h" +#include "erl_threads.h" #if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS) # define ERTS_PROC_LOCK_OWN_IMPL 0 @@ -73,18 +73,24 @@ typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_t_ { #if ERTS_PROC_LOCK_OWN_IMPL #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic32_t flags; + erts_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt_main; - erts_lcnt_lock_t lcnt_link; - erts_lcnt_lock_t lcnt_msgq; - erts_lcnt_lock_t lcnt_btm; - erts_lcnt_lock_t lcnt_status; - erts_lcnt_lock_t lcnt_trace; +#if defined(ERTS_ENABLE_LOCK_COUNT) && !ERTS_PROC_LOCK_RAW_MUTEX_IMPL + /* Each erts_mtx_t has its own lock counter ^ */ + + #define ERTS_LCNT_PROCLOCK_IDX_MAIN 0 + #define ERTS_LCNT_PROCLOCK_IDX_LINK 1 + #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 2 + #define ERTS_LCNT_PROCLOCK_IDX_BTM 3 + #define ERTS_LCNT_PROCLOCK_IDX_STATUS 4 + #define ERTS_LCNT_PROCLOCK_IDX_TRACE 5 + + #define ERTS_LCNT_PROCLOCK_COUNT 6 + + erts_lcnt_ref_t lcnt_carrier; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; @@ -97,7 +103,7 @@ typedef struct erts_proc_lock_t_ { # error "no implementation" #endif #ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; + erts_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif } erts_proc_lock_t; @@ -237,32 +243,188 @@ typedef struct erts_proc_lock_t_ { /* Lock counter implemetation */ #ifdef ERTS_ENABLE_LOCK_POSITION -#define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__) -#define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__) +#define erts_proc_lock__(P,I,L) erts_proc_lock_x__(P,I,L,__FILE__,__LINE__) +#define erts_proc_lock(P,L) erts_proc_lock_x(P,L,__FILE__,__LINE__) #endif -#if defined(ERTS_SMP) && defined (ERTS_ENABLE_LOCK_COUNT) +#if defined (ERTS_ENABLE_LOCK_COUNT) void erts_lcnt_proc_lock_init(Process *p); void erts_lcnt_proc_lock_destroy(Process *p); + +ERTS_GLB_INLINE void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line); -void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res); -void erts_lcnt_enable_proc_lock_count(int enable); +void erts_lcnt_enable_proc_lock_count(Process *proc, int enable); +void erts_lcnt_update_process_locks(int enable); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, + char *file, unsigned int line) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, file, line); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, file, line); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, file, line); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, file, line); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, file, line); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, file, line); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, res); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, res); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, res); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, res); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, res); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, res); + } + + erts_lcnt_close_ref(handle, carrier); + } +} /* reversed logic */ +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ERTS_ENABLE_LOCK_COUNT*/ /* --- Process lock checking ----------------------------------------------- */ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) -#define ERTS_SMP_CHK_NO_PROC_LOCKS \ +#if defined(ERTS_ENABLE_LOCK_CHECK) +#define ERTS_CHK_NO_PROC_LOCKS \ erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__) -#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \ +#define ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \ erts_proc_lc_chk_only_proc_main((P)) void erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line); @@ -281,8 +443,8 @@ void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char* file, unsigned int line); void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); #else -#define ERTS_SMP_CHK_NO_PROC_LOCKS -#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) +#define ERTS_CHK_NO_PROC_LOCKS +#define ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) #endif #endif /* #ifndef ERTS_PROC_LOCK_LOCK_CHECK__ */ @@ -293,7 +455,6 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); #ifndef ERTS_PROCESS_LOCK_H__ #define ERTS_PROCESS_LOCK_H__ -#ifdef ERTS_SMP typedef struct { union { @@ -310,21 +471,21 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \ + ((ErtsProcLocks) erts_atomic32_read_band_nob(&(L)->flags, \ (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \ + ((ErtsProcLocks) erts_atomic32_read_bor_acqb(&(L)->flags, \ (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ + ((ErtsProcLocks) erts_atomic32_cmpxchg_acqb(&(L)->flags, \ (erts_aint32_t) (NEW), \ (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \ + ((ErtsProcLocks) erts_atomic32_cmpxchg_relb(&(L)->flags, \ (erts_aint32_t) (NEW), \ (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags)) + ((ErtsProcLocks) erts_atomic32_read_nob(&(L)->flags)) #else /* no opt atomic ops */ @@ -395,22 +556,22 @@ ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *); ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *); ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *); -ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, +ERTS_GLB_INLINE ErtsProcLocks erts_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, +ERTS_GLB_INLINE void erts_proc_lock_x__(Process *, erts_pix_lock_t *, ErtsProcLocks, char *file, unsigned int line); #else -ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, +ERTS_GLB_INLINE void erts_proc_lock__(Process *, erts_pix_lock_t *, ErtsProcLocks); #endif -ERTS_GLB_INLINE void erts_smp_proc_unlock__(Process *, +ERTS_GLB_INLINE void erts_proc_unlock__(Process *, erts_pix_lock_t *, ErtsProcLocks); -ERTS_GLB_INLINE int erts_smp_proc_trylock__(Process *, +ERTS_GLB_INLINE int erts_proc_trylock__(Process *, erts_pix_lock_t *, ErtsProcLocks); @@ -438,7 +599,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) } /* - * Helper function for erts_smp_proc_lock__ and erts_smp_proc_trylock__. + * Helper function for erts_proc_lock__ and erts_proc_trylock__. * * Attempts to grab all of 'locks' simultaneously. * @@ -451,7 +612,7 @@ ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) * Does not release the pix lock. */ ERTS_GLB_INLINE ErtsProcLocks -erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) +erts_proc_raw_trylock__(Process *p, ErtsProcLocks locks) { #if ERTS_PROC_LOCK_OWN_IMPL ErtsProcLocks expct_lflgs = 0; @@ -520,12 +681,12 @@ busy_main: ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_proc_lock_x__(Process *p, +erts_proc_lock_x__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks, char *file, unsigned int line) #else -erts_smp_proc_lock__(Process *p, +erts_proc_lock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) #endif @@ -547,7 +708,7 @@ erts_smp_proc_lock__(Process *p, erts_proc_lc_lock(p, locks, file, line); #endif - old_lflgs = erts_smp_proc_raw_trylock__(p, locks); + old_lflgs = erts_proc_raw_trylock__(p, locks); if (old_lflgs != 0) { /* @@ -599,7 +760,7 @@ erts_smp_proc_lock__(Process *p, } ERTS_GLB_INLINE void -erts_smp_proc_unlock__(Process *p, +erts_proc_unlock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { @@ -692,7 +853,7 @@ erts_smp_proc_unlock__(Process *p, } ERTS_GLB_INLINE int -erts_smp_proc_trylock__(Process *p, +erts_proc_trylock__(Process *p, erts_pix_lock_t *pix_lck, ErtsProcLocks locks) { @@ -713,7 +874,7 @@ erts_smp_proc_trylock__(Process *p, erts_pix_lock(pix_lck); #endif - if (erts_smp_proc_raw_trylock__(p, locks) != 0) { + if (erts_proc_raw_trylock__(p, locks) != 0) { /* Didn't get all locks... */ res = EBUSY; @@ -750,7 +911,7 @@ erts_smp_proc_trylock__(Process *p, return res; #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL - if (erts_smp_proc_raw_trylock__(p, locks) != 0) + if (erts_proc_raw_trylock__(p, locks) != 0) return EBUSY; else { #ifdef ERTS_PROC_LOCK_DEBUG @@ -771,11 +932,11 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) if (locks & lock) { erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]); + lock_count = erts_atomic32_inc_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]); + lock_count = erts_atomic32_dec_read_nob(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } @@ -785,15 +946,14 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* ERTS_SMP */ #ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line); +ERTS_GLB_INLINE void erts_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line); #else -ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks); +ERTS_GLB_INLINE void erts_proc_lock(Process *, ErtsProcLocks); #endif -ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks); -ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks); +ERTS_GLB_INLINE void erts_proc_unlock(Process *, ErtsProcLocks); +ERTS_GLB_INLINE int erts_proc_trylock(Process *, ErtsProcLocks); ERTS_GLB_INLINE void erts_proc_inc_refc(Process *); ERTS_GLB_INLINE void erts_proc_dec_refc(Process *); @@ -804,79 +964,65 @@ ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *); ERTS_GLB_INLINE void #ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line) +erts_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line) #else -erts_smp_proc_lock(Process *p, ErtsProcLocks locks) +erts_proc_lock(Process *p, ErtsProcLocks locks) #endif { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_smp_proc_lock_x__(p, +#if defined(ERTS_ENABLE_LOCK_POSITION) + erts_proc_lock_x__(p, #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else ERTS_PID2PIXLOCK(p->common.id), #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/ locks, file, line); -#elif defined(ERTS_SMP) - erts_smp_proc_lock__(p, +#else + erts_proc_lock__(p, #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else ERTS_PID2PIXLOCK(p->common.id), #endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/ locks); -#endif /*ERTS_SMP*/ +#endif /*ERTS_ENABLE_LOCK_POSITION*/ } ERTS_GLB_INLINE void -erts_smp_proc_unlock(Process *p, ErtsProcLocks locks) +erts_proc_unlock(Process *p, ErtsProcLocks locks) { -#ifdef ERTS_SMP - erts_smp_proc_unlock__(p, + erts_proc_unlock__(p, #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else ERTS_PID2PIXLOCK(p->common.id), #endif locks); -#endif } ERTS_GLB_INLINE int -erts_smp_proc_trylock(Process *p, ErtsProcLocks locks) +erts_proc_trylock(Process *p, ErtsProcLocks locks) { -#ifndef ERTS_SMP - return 0; -#else - return erts_smp_proc_trylock__(p, + return erts_proc_trylock__(p, #if ERTS_PROC_LOCK_ATOMIC_IMPL NULL, #else ERTS_PID2PIXLOCK(p->common.id), #endif locks); -#endif } ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p) { - ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); -#ifdef ERTS_SMP + ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); erts_ptab_atmc_inc_refc(&p->common); -#else - erts_ptab_inc_refc(&p->common); -#endif } ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) { Sint referred; - ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); -#ifdef ERTS_SMP + ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); referred = erts_ptab_atmc_dec_test_refc(&p->common); -#else - referred = erts_ptab_dec_test_refc(&p->common); -#endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); erts_free_proc(p); @@ -886,12 +1032,8 @@ ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p) ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) { Sint referred; - ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); -#ifdef ERTS_SMP + ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc); -#else - referred = erts_ptab_add_test_refc(&p->common, add_refc); -#endif if (!referred) { ASSERT(ERTS_PROC_IS_EXITING(p)); erts_free_proc(p); @@ -900,17 +1042,12 @@ ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc) ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p) { - ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); -#ifdef ERTS_SMP + ASSERT(!(erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY)); return erts_ptab_atmc_read_refc(&p->common); -#else - return erts_ptab_read_refc(&p->common); -#endif } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#ifdef ERTS_SMP void erts_proc_lock_init(Process *); void erts_proc_lock_fin(Process *); void erts_proc_safelock(Process *a_proc, @@ -919,7 +1056,6 @@ void erts_proc_safelock(Process *a_proc, Process *b_proc, ErtsProcLocks b_have_locks, ErtsProcLocks b_need_locks); -#endif /* * --- Process table lookup ------------------------------------------------ @@ -951,9 +1087,6 @@ ERTS_GLB_INLINE Process *erts_pix2proc(int ix); ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid); -#ifndef ERTS_SMP -ERTS_GLB_INLINE -#endif Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -970,7 +1103,7 @@ ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid) { Process *proc; - ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + ERTS_LC_ASSERT(erts_thr_progress_lc_is_delaying()); if (is_not_internal_pid(pid)) return NULL; @@ -990,25 +1123,6 @@ ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid) return proc; } -#ifndef ERTS_SMP -ERTS_GLB_INLINE Process * -erts_pid2proc_opt(Process *c_p_unused, - ErtsProcLocks c_p_have_locks_unused, - Eterm pid, - ErtsProcLocks pid_need_locks_unused, - int flags) -{ - Process *proc = erts_proc_lookup_raw(pid); - if (!proc) - return NULL; - if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X) - && ERTS_PROC_IS_EXITING(proc)) - return NULL; - if (flags & ERTS_P2P_FLG_INC_REFC) - erts_proc_inc_refc(proc); - return proc; -} -#endif /* !ERTS_SMP */ #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index c3d59cb3a8..38c095fb4a 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -284,31 +284,31 @@ struct ErtsPTabListBifData_ { static ERTS_INLINE void last_data_init_nob(ErtsPTab *ptab, Uint64 val) { - erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val); + erts_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE void last_data_set_relb(ErtsPTab *ptab, Uint64 val) { - erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val); + erts_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE Uint64 last_data_read_nob(ErtsPTab *ptab) { - return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data); + return (Uint64) erts_atomic64_read_nob(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_read_acqb(ErtsPTab *ptab) { - return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data); + return (Uint64) erts_atomic64_read_acqb(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) { - return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data, + return (Uint64) erts_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data, (erts_aint64_t) new, (erts_aint64_t) exp); } @@ -346,9 +346,9 @@ ix_to_free_id_data_ix(ErtsPTab *ptab, Uint32 ix) UWord erts_ptab_mem_size(ErtsPTab *ptab) { - UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t); + UWord size = ptab->r.o.max*sizeof(erts_atomic_t); if (ptab->r.o.free_id_data) - size += ptab->r.o.max*sizeof(erts_smp_atomic32_t); + size += ptab->r.o.max*sizeof(erts_atomic32_t); return size; } @@ -367,13 +367,14 @@ erts_ptab_init_table(ErtsPTab *ptab, size_t tab_sz, alloc_sz; Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; char *tab_end; - erts_smp_atomic_t *tab_entry; - erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; - - erts_smp_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name); - erts_smp_atomic32_init_nob(&ptab->vola.tile.count, 0); + erts_atomic_t *tab_entry; + erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED; + + erts_rwmtx_init_opt(&ptab->list.data.rwmtx, &rwmtx_opts, name, NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + erts_atomic32_init_nob(&ptab->vola.tile.count, 0); last_data_init_nob(ptab, ~((Uint64) 0)); /* A size that is a power of 2 is to prefer performance wise */ @@ -387,20 +388,20 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->r.o.element_size = element_size; ptab->r.o.max = size; - tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t)); + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic_t)); alloc_sz = tab_sz; if (!legacy) - alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic32_t)); ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz); tab_end = ((char *) ptab->r.o.tab) + tab_sz; tab_entry = ptab->r.o.tab; while (tab_end > ((char *) tab_entry)) { - erts_smp_atomic_init_nob(tab_entry, ERTS_AINT_NULL); + erts_atomic_init_nob(tab_entry, ERTS_AINT_NULL); tab_entry++; } tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; - ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t)); + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_atomic_t)); ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */ ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ @@ -428,11 +429,11 @@ erts_ptab_init_table(ErtsPTab *ptab, } else { - tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); - ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end; + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_atomic32_t)); + ptab->r.o.free_id_data = (erts_atomic32_t *) tab_end; tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; - ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t)); + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_atomic32_t)); ptab->r.o.dix_cl_mask = tab_cache_lines-1; ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); @@ -447,19 +448,19 @@ erts_ptab_init_table(ErtsPTab *ptab, ix = 0; for (cl = 0; cl < tab_cache_lines; cl++) { for (cli = 0; cli < ix_per_cache_line; cli++) { - erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix], + erts_atomic32_init_nob(&ptab->r.o.free_id_data[ix], cli*tab_cache_lines+cl); - ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data); + ASSERT(erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data); ix++; } } - erts_smp_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1); - erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1); + erts_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1); + erts_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1); } - erts_smp_interval_init(&ptab->list.data.interval); + erts_interval_init(&ptab->list.data.interval); ptab->list.data.deleted.start = NULL; ptab->list.data.deleted.end = NULL; ptab->list.data.chunks = (((ptab->r.o.max - 1) @@ -479,9 +480,9 @@ erts_ptab_init_table(ErtsPTab *ptab, * have ERTS_PTAB_MAX_SIZE-1 valid elements in the table while * still having a table size of the power of 2. */ - erts_smp_atomic32_inc_nob(&ptab->vola.tile.count); + erts_atomic32_inc_nob(&ptab->vola.tile.count); pix = erts_ptab_data2pix(ptab, ptab->r.o.invalid_data); - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], + erts_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab->r.o.invalid_element); } @@ -505,12 +506,12 @@ erts_ptab_new_element(ErtsPTab *ptab, erts_ptab_rlock(ptab); - count = erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.count); + count = erts_atomic32_inc_read_acqb(&ptab->vola.tile.count); if (count > ptab->r.o.max) { while (1) { erts_aint32_t act_count; - act_count = erts_smp_atomic32_cmpxchg_relb(&ptab->vola.tile.count, + act_count = erts_atomic32_cmpxchg_relb(&ptab->vola.tile.count, count-1, count); if (act_count == count) { @@ -524,14 +525,14 @@ erts_ptab_new_element(ErtsPTab *ptab, } ptab_el->u.alive.started_interval - = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + = erts_current_interval_nob(erts_ptab_interval(ptab)); if (ptab->r.o.free_id_data) { do { - ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix); + ix = (Uint32) erts_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix); ix = ix_to_free_id_data_ix(ptab, ix); - data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix], + data = erts_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix], (erts_aint32_t)ptab->r.o.invalid_data); }while ((Eterm)data == ptab->r.o.invalid_data); @@ -545,10 +546,10 @@ erts_ptab_new_element(ErtsPTab *ptab, pix = erts_ptab_data2pix(ptab, (Eterm) data); #ifdef DEBUG - ASSERT(ERTS_AINT_NULL == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + ASSERT(ERTS_AINT_NULL == erts_atomic_xchg_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el)); #else - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); + erts_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); #endif erts_ptab_runlock(ptab); @@ -562,7 +563,7 @@ erts_ptab_new_element(ErtsPTab *ptab, restart: ptab_el->u.alive.started_interval - = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + = erts_current_interval_nob(erts_ptab_interval(ptab)); ld = last_data_read_acqb(ptab); @@ -570,10 +571,10 @@ erts_ptab_new_element(ErtsPTab *ptab, while (1) { ld++; pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); - if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) + if (erts_atomic_read_nob(&ptab->r.o.tab[pix]) == ERTS_AINT_NULL) { erts_aint_t val; - val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], + val = erts_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], invalid, ERTS_AINT_NULL); @@ -620,10 +621,10 @@ erts_ptab_new_element(ErtsPTab *ptab, /* Move into slot reserved */ #ifdef DEBUG - ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + ASSERT(invalid == erts_atomic_xchg_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el)); #else - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); + erts_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); #endif if (rlocked) @@ -643,7 +644,7 @@ save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) sizeof(ErtsPTabDeletedElement)); ERTS_PTAB_LIST_ASSERT(ptab->list.data.deleted.start && ptab->list.data.deleted.end); - ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab)); ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); @@ -653,7 +654,7 @@ save_deleted_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) ptdep->u.element.id = ptab_el->id; ptdep->u.element.inserted = ptab_el->u.alive.started_interval; ptdep->u.element.deleted = - erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + erts_current_interval_nob(erts_ptab_interval(ptab)); ptab->list.data.deleted.end->next = ptdep; ptab->list.data.deleted.end = ptdep; @@ -677,7 +678,7 @@ erts_ptab_delete_element(ErtsPTab *ptab, pix = erts_ptab_id2pix(ptab, ptab_el->id); /* *Need* to be an managed thread */ - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread()); + ERTS_LC_ASSERT(erts_thr_progress_is_managed_thread()); erts_ptab_rlock(ptab); maybe_save = ptab->list.data.deleted.end != NULL; @@ -686,7 +687,7 @@ erts_ptab_delete_element(ErtsPTab *ptab, erts_ptab_rwlock(ptab); } - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL); + erts_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL); if (ptab->r.o.free_id_data) { Uint32 prev_data; @@ -702,17 +703,17 @@ erts_ptab_delete_element(ErtsPTab *ptab, ASSERT(pix == erts_ptab_data2pix(ptab, data)); do { - ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix); + ix = (Uint32) erts_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix); ix = ix_to_free_id_data_ix(ptab, ix); - prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix], + prev_data = erts_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix], data, ptab->r.o.invalid_data); }while ((Eterm)prev_data != ptab->r.o.invalid_data); } - ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0); - erts_smp_atomic32_dec_relb(&ptab->vola.tile.count); + ASSERT(erts_atomic32_read_nob(&ptab->vola.tile.count) > 0); + erts_atomic32_dec_relb(&ptab->vola.tile.count); if (!maybe_save) erts_ptab_runlock(ptab); @@ -926,7 +927,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) sizeof(ErtsPTabDeletedElement)); ptlbdp->bif_invocation->ix = -1; ptlbdp->bif_invocation->u.bif_invocation.interval - = erts_smp_step_interval_nob(erts_ptab_interval(ptab)); + = erts_step_interval_nob(erts_ptab_interval(ptab)); ERTS_PTAB_LIST_DBG_CHK_DEL_LIST(ptab); ptlbdp->bif_invocation->next = NULL; @@ -967,12 +968,12 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) locked = 1; } - ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab)); ERTS_PTAB_LIST_DBG_TRACE(p->common.id, insp_table); if (cix != 0) ptlbdp->chunk[cix].interval - = erts_smp_step_interval_nob(erts_ptab_interval(ptab)); + = erts_step_interval_nob(erts_ptab_interval(ptab)); else if (ptlbdp->bif_invocation) ptlbdp->chunk[0].interval = *invocation_interval_p; /* else: interval is irrelevant */ @@ -1330,18 +1331,18 @@ static void assert_ptab_consistency(ErtsPTab *ptab) int null_slots = 0; for (ix=0; ix < ptab->r.o.max; ix++) { - if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) { + if (erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) { ++free_pids; - data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]); + data = erts_atomic32_read_nob(&ptab->r.o.free_id_data[ix]); pix = erts_ptab_data2pix(ptab, (Eterm) data); ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL); } - if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) { + if (erts_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) { ++null_slots; } } ASSERT(free_pids == null_slots); - ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count)); + ASSERT(free_pids == ptab->r.o.max - erts_atomic32_read_nob(&ptab->vola.tile.count)); } #endif } @@ -1365,7 +1366,7 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) Uint32 i, max_ix, num, stop_id_ix; max_ix = ptab->r.o.max - 1; num = next; - id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix); + id_ix = (Uint32) erts_atomic32_read_nob(&ptab->vola.tile.aid_ix); for (i=0; i <= max_ix; ++i) { Uint32 pix; @@ -1379,26 +1380,26 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { ++id_ix; dix = ix_to_free_id_data_ix(ptab, id_ix); - erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num); + erts_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num); ASSERT(pix == erts_ptab_data2pix(ptab, num)); } } - erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix); + erts_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix); /* Write invalid_data in rest of free_id_data[]: */ - stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix; + stop_id_ix = (1 + erts_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix; while (1) { id_ix = (id_ix+1) & max_ix; if (id_ix == stop_id_ix) break; dix = ix_to_free_id_data_ix(ptab, id_ix); - erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], + erts_atomic32_set_nob(&ptab->r.o.free_id_data[dix], ptab->r.o.invalid_data); } } - id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1; + id_ix = (Uint32) erts_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1; dix = ix_to_free_id_data_ix(ptab, id_ix); - res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]); + res = (Sint) erts_atomic32_read_nob(&ptab->r.o.free_id_data[dix]); } else { /* Deprecated legacy algorithm... */ @@ -1615,11 +1616,11 @@ debug_ptab_list_verify_all_pids(ErtsPTabListBifData *ptlbdp) static void debug_ptab_list_check_del_list(ErtsPTab *ptab) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_ptab_is_rwlocked(ptab)); + ERTS_LC_ASSERT(erts_lc_ptab_is_rwlocked(ptab)); if (!ptab->list.data.deleted.start) ERTS_PTAB_LIST_ASSERT(!ptab->list.data.deleted.end); else { - Uint64 curr_interval = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + Uint64 curr_interval = erts_current_interval_nob(erts_ptab_interval(ptab)); Uint64 *prev_x_interval_p = NULL; ErtsPTabDeletedElement *ptdep; diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index fecfd96ab0..4858cc8ab8 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -60,7 +60,7 @@ typedef struct { } refc; ErtsTracer tracer; Uint trace_flags; - erts_smp_atomic_t timer; + erts_atomic_t timer; union { /* --- While being alive --- */ struct { @@ -78,7 +78,7 @@ typedef struct { typedef struct ErtsPTabDeletedElement_ ErtsPTabDeletedElement; typedef struct { - erts_smp_rwmtx_t rwmtx; + erts_rwmtx_t rwmtx; erts_interval_t interval; struct { ErtsPTabDeletedElement *start; @@ -88,15 +88,15 @@ typedef struct { } ErtsPTabListData; typedef struct { - erts_smp_atomic64_t last_data; - erts_smp_atomic32_t count; - erts_smp_atomic32_t aid_ix; - erts_smp_atomic32_t fid_ix; + erts_atomic64_t last_data; + erts_atomic32_t count; + erts_atomic32_t aid_ix; + erts_atomic32_t fid_ix; } ErtsPTabVolatileData; typedef struct { - erts_smp_atomic_t *tab; - erts_smp_atomic32_t *free_id_data; + erts_atomic_t *tab; + erts_atomic32_t *free_id_data; Uint32 max; Uint32 pix_mask; Uint32 pix_cl_mask; @@ -223,8 +223,8 @@ ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab); ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab); ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab); -ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab); -ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_lc_ptab_is_rlocked(ErtsPTab *ptab); +ERTS_GLB_INLINE int erts_lc_ptab_is_rwlocked(ErtsPTab *ptab); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -245,7 +245,7 @@ ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab) { int max = ptab->r.o.max; - erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count); + erts_aint32_t res = erts_atomic32_read_nob(&ptab->vola.tile.count); if (max == ERTS_PTAB_MAX_SIZE) { max--; res--; @@ -352,25 +352,25 @@ erts_ptab_id2data(ErtsPTab *ptab, Eterm id) ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix) { ASSERT(0 <= ix && ix < ptab->r.o.max); - return erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]); + return erts_atomic_read_nob(&ptab->r.o.tab[ix]); } ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix) { ASSERT(0 <= ix && ix < ptab->r.o.max); - return erts_smp_atomic_read_ddrb(&ptab->r.o.tab[ix]); + return erts_atomic_read_ddrb(&ptab->r.o.tab[ix]); } ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix) { ASSERT(0 <= ix && ix < ptab->r.o.max); - return erts_smp_atomic_read_rb(&ptab->r.o.tab[ix]); + return erts_atomic_read_rb(&ptab->r.o.tab[ix]); } ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix) { ASSERT(0 <= ix && ix < ptab->r.o.max); - return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]); + return erts_atomic_read_acqb(&ptab->r.o.tab[ix]); } ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el) @@ -386,11 +386,9 @@ ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el) ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el) { erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc); - ERTS_SMP_LC_ASSERT(refc >= 0); -#ifdef ERTS_SMP + ERTS_LC_ASSERT(refc >= 0); if (refc == 0) ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); -#endif return (Sint) refc; } @@ -399,7 +397,7 @@ ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el { erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc, (erts_aint_t) add_refc); - ERTS_SMP_LC_ASSERT(refc >= 0); + ERTS_LC_ASSERT(refc >= 0); return (Sint) refc; } @@ -417,7 +415,7 @@ ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el) ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el) { Sint refc = --ptab_el->refc.sint; - ERTS_SMP_LC_ASSERT(refc >= 0); + ERTS_LC_ASSERT(refc >= 0); return refc; } @@ -425,7 +423,7 @@ ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el, Sint add_refc) { ptab_el->refc.sint += add_refc; - ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0); + ERTS_LC_ASSERT(ptab_el->refc.sint >= 0); return (Sint) ptab_el->refc.sint; } @@ -436,42 +434,42 @@ ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el) ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab) { - erts_smp_rwmtx_rlock(&ptab->list.data.rwmtx); + erts_rwmtx_rlock(&ptab->list.data.rwmtx); } ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab) { - return erts_smp_rwmtx_tryrlock(&ptab->list.data.rwmtx); + return erts_rwmtx_tryrlock(&ptab->list.data.rwmtx); } ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab) { - erts_smp_rwmtx_runlock(&ptab->list.data.rwmtx); + erts_rwmtx_runlock(&ptab->list.data.rwmtx); } ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab) { - erts_smp_rwmtx_rwlock(&ptab->list.data.rwmtx); + erts_rwmtx_rwlock(&ptab->list.data.rwmtx); } ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab) { - return erts_smp_rwmtx_tryrwlock(&ptab->list.data.rwmtx); + return erts_rwmtx_tryrwlock(&ptab->list.data.rwmtx); } ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab) { - erts_smp_rwmtx_rwunlock(&ptab->list.data.rwmtx); + erts_rwmtx_rwunlock(&ptab->list.data.rwmtx); } -ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab) +ERTS_GLB_INLINE int erts_lc_ptab_is_rlocked(ErtsPTab *ptab) { - return erts_smp_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx); + return erts_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx); } -ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab) +ERTS_GLB_INLINE int erts_lc_ptab_is_rwlocked(ErtsPTab *ptab) { - return erts_smp_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx); + return erts_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx); } #endif diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index cab4bd73db..ab204303d7 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -32,13 +32,12 @@ # include "config.h" #endif -#ifdef ERTS_SMP #include "erl_process.h" #include "erl_thr_progress.h" erts_sspa_data_t * -erts_sspa_create(size_t blk_sz, int pa_size) +erts_sspa_create(size_t blk_sz, int pa_size, int nthreads, const char* name) { erts_sspa_data_t *data; size_t tot_size; @@ -49,22 +48,30 @@ erts_sspa_create(size_t blk_sz, int pa_size) int no_blocks = pa_size; int no_blocks_per_chunk; - if (erts_no_schedulers == 1) + if (!name) { /* schedulers only variant */ + ASSERT(!nthreads); + nthreads = erts_no_schedulers; + } + else { + ASSERT(nthreads > 0); + } + + if (nthreads == 1) no_blocks_per_chunk = no_blocks; else { int extra = (no_blocks - 1)/4 + 1; if (extra == 0) extra = 1; no_blocks_per_chunk = no_blocks; - no_blocks_per_chunk += extra*erts_no_schedulers; - no_blocks_per_chunk /= erts_no_schedulers; + no_blocks_per_chunk += extra * nthreads; + no_blocks_per_chunk /= nthreads; } - no_blocks = no_blocks_per_chunk * erts_no_schedulers; + no_blocks = no_blocks_per_chunk * nthreads; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t)); chunk_mem_size += blk_sz * no_blocks_per_chunk; chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size); tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t)); - tot_size += chunk_mem_size*erts_no_schedulers; + tot_size += chunk_mem_size * nthreads; p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size); data = (erts_sspa_data_t *) p; @@ -73,10 +80,16 @@ erts_sspa_create(size_t blk_sz, int pa_size) data->chunks_mem_size = chunk_mem_size; data->start = chunk_start; - data->end = chunk_start + chunk_mem_size*erts_no_schedulers; + data->end = chunk_start + chunk_mem_size * nthreads; + data->nthreads = nthreads; + + if (name) { /* thread variant */ + erts_tsd_key_create(&data->tsd_key, (char*)name); + erts_atomic_init_nob(&data->id_generator, 0); + } /* Initialize all chunks */ - for (cix = 0; cix < erts_no_schedulers; cix++) { + for (cix = 0; cix < nthreads; cix++) { erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix); erts_sspa_chunk_header_t *chdr = &chnk->aligned.header; erts_sspa_blk_t *blk; @@ -161,7 +174,7 @@ enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr, if ((i & 1) == 0) itmp = itmp2; else { - enq = (erts_sspa_blk_t *) itmp; + enq = (erts_sspa_blk_t *) itmp2; itmp = erts_atomic_read_acqb(&enq->next_atmc); ASSERT(itmp != ERTS_AINT_NULL); } @@ -325,4 +338,3 @@ erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr, return res; } -#endif /* ERTS_SMP */ diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h index 7808d7d438..d232db0e69 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.h +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h @@ -31,7 +31,6 @@ #ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__ #define ERTS_SCHED_SPEC_PRE_ALLOC_H__ -#ifdef ERTS_SMP #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #define ERL_THR_PROGRESS_TSD_TYPE_ONLY @@ -60,6 +59,11 @@ typedef struct { char *start; char *end; int chunks_mem_size; + int nthreads; + + /* Used only by thread variant: */ + erts_tsd_key_t tsd_key; + erts_atomic_t id_generator; } erts_sspa_data_t; typedef union erts_sspa_blk_t_ erts_sspa_blk_t; @@ -141,7 +145,9 @@ check_local_list(erts_sspa_chunk_header_t *chdr) #endif erts_sspa_data_t *erts_sspa_create(size_t blk_sz, - int pa_size); + int pa_size, + int nthreads, + const char* name); void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr, erts_sspa_blk_t *blk, int cinit); @@ -159,7 +165,7 @@ ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk); ERTS_GLB_INLINE erts_sspa_chunk_t * erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix) { - ASSERT(0 <= cix && cix < erts_no_schedulers); + ASSERT(0 <= cix && cix < data->nthreads); return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size); } @@ -172,7 +178,7 @@ erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr) return -1; diff = ((char *) ptr) - data->start; cix = (int) diff / data->chunks_mem_size; - ASSERT(0 <= cix && cix < erts_no_schedulers); + ASSERT(0 <= cix && cix < data->nthreads); return cix; } @@ -236,6 +242,5 @@ erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk) #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#endif /* ERTS_SMP */ #endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */ diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h deleted file mode 100644 index 181736b009..0000000000 --- a/erts/emulator/beam/erl_smp.h +++ /dev/null @@ -1,1640 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ -/* - * SMP interface to ethread library. - * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h", - * plus changes to NOP operations when ERTS_SMP is disabled. - * Author: Mikael Pettersson - */ -#ifndef ERL_SMP_H -#define ERL_SMP_H -#include "erl_threads.h" - -#ifdef ERTS_ENABLE_LOCK_POSITION -#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__) -#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__) -#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__) -#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__) -#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__) -#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__) -#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__) -#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__) -#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__) -#endif - - -#ifdef ERTS_SMP -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER -typedef erts_thr_opts_t erts_smp_thr_opts_t; -typedef erts_thr_init_data_t erts_smp_thr_init_data_t; -typedef erts_tid_t erts_smp_tid_t; -typedef erts_mtx_t erts_smp_mtx_t; -typedef erts_cnd_t erts_smp_cnd_t; -#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER -#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL -#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ -#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ - ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ -#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED -#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED -#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED -typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; -typedef erts_rwmtx_t erts_smp_rwmtx_t; -typedef erts_tsd_key_t erts_smp_tsd_key_t; -#define erts_smp_dw_atomic_t erts_dw_atomic_t -#define erts_smp_atomic_t erts_atomic_t -#define erts_smp_atomic32_t erts_atomic32_t -#define erts_smp_atomic64_t erts_atomic64_t -typedef erts_spinlock_t erts_smp_spinlock_t; -typedef erts_rwlock_t erts_smp_rwlock_t; -void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ - -#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER -#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER -#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER -#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER - -#else /* #ifdef ERTS_SMP */ - -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} -typedef int erts_smp_thr_opts_t; -typedef int erts_smp_thr_init_data_t; -typedef int erts_smp_tid_t; -typedef int erts_smp_mtx_t; -typedef int erts_smp_cnd_t; -#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} -#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 -#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 -#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 -#define ERTS_SMP_RWMTX_LONG_LIVED 0 -#define ERTS_SMP_RWMTX_SHORT_LIVED 0 -#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 -typedef struct { - char type; - char lived; - int main_spincount; - int aux_spincount; -} erts_smp_rwmtx_opt_t; -typedef int erts_smp_rwmtx_t; -typedef int erts_smp_tsd_key_t; -#define erts_smp_dw_atomic_t erts_no_dw_atomic_t -#define erts_smp_atomic_t erts_no_atomic_t -#define erts_smp_atomic32_t erts_no_atomic32_t -#define erts_smp_atomic64_t erts_no_atomic64_t -#if __GNUC__ > 2 -typedef struct { } erts_smp_spinlock_t; -typedef struct { } erts_smp_rwlock_t; -#else -typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; -typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; -#endif - -#define ERTS_SMP_MEMORY_BARRIER -#define ERTS_SMP_WRITE_MEMORY_BARRIER -#define ERTS_SMP_READ_MEMORY_BARRIER -#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER - -#endif /* #ifdef ERTS_SMP */ - -ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); -ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid, - void * (*func)(void *), - void *arg, - erts_smp_thr_opts_t *opts); -ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res); -ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid); -ERTS_GLB_INLINE void erts_smp_thr_exit(void *res); -ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void)); -ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void); -ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -#define ERTS_SMP_HAVE_REC_MTX_INIT 1 -ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx); -#endif -ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); -ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); -ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -#ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line); -ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line); -#else -ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx); -#endif -ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd); -ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd); -ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, - erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); -ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); -ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); -ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, - erts_smp_rwmtx_opt_t *opt, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, - erts_smp_rwmtx_opt_t *opt, - char *name); -ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, - char *name); -ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); -#ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); -ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); -ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line); -#else -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); -#endif -ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); -ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock, - char *name); -ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock); -ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line); -#else -ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock); -#endif -ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock); -ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock, - char *name); -ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock); -#ifdef ERTS_ENABLE_LOCK_POSITION -ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line); -ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line); -#else -ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock); -#endif -ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, - char *keyname); -ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); -ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); - -#ifdef ERTS_THR_HAVE_SIG_FUNCS -#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 -ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how, - const sigset_t *set, - sigset_t *oset); -ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); -#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ - -/* - * See "Documentation of atomics and memory barriers" at the top - * of erl_threads.h for info on atomics. - */ - -#ifdef ERTS_SMP - -/* Double word size atomics */ - -#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob -#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob -#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob -#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob - -#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb -#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb -#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb -#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb - -#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb -#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb -#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb -#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb - -#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb -#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb -#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb -#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb - -#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb -#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb -#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb -#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb - -#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb -#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb -#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb -#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb - -#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb -#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb -#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb -#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb - -#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty -#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty - -/* Word size atomics */ - -#define erts_smp_atomic_init_nob erts_atomic_init_nob -#define erts_smp_atomic_set_nob erts_atomic_set_nob -#define erts_smp_atomic_read_nob erts_atomic_read_nob -#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob -#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob -#define erts_smp_atomic_inc_nob erts_atomic_inc_nob -#define erts_smp_atomic_dec_nob erts_atomic_dec_nob -#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob -#define erts_smp_atomic_add_nob erts_atomic_add_nob -#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob -#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob -#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob -#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob -#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob - -#define erts_smp_atomic_init_mb erts_atomic_init_mb -#define erts_smp_atomic_set_mb erts_atomic_set_mb -#define erts_smp_atomic_read_mb erts_atomic_read_mb -#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb -#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb -#define erts_smp_atomic_inc_mb erts_atomic_inc_mb -#define erts_smp_atomic_dec_mb erts_atomic_dec_mb -#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb -#define erts_smp_atomic_add_mb erts_atomic_add_mb -#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb -#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb -#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb -#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb -#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb - -#define erts_smp_atomic_init_acqb erts_atomic_init_acqb -#define erts_smp_atomic_set_acqb erts_atomic_set_acqb -#define erts_smp_atomic_read_acqb erts_atomic_read_acqb -#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb -#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb -#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb -#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb -#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb -#define erts_smp_atomic_add_acqb erts_atomic_add_acqb -#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb -#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb -#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb -#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb -#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb - -#define erts_smp_atomic_init_relb erts_atomic_init_relb -#define erts_smp_atomic_set_relb erts_atomic_set_relb -#define erts_smp_atomic_read_relb erts_atomic_read_relb -#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb -#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb -#define erts_smp_atomic_inc_relb erts_atomic_inc_relb -#define erts_smp_atomic_dec_relb erts_atomic_dec_relb -#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb -#define erts_smp_atomic_add_relb erts_atomic_add_relb -#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb -#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb -#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb -#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb -#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb - -#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb -#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb -#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb -#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb -#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb -#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb -#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb -#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb -#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb -#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb -#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb -#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb -#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb -#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb - -#define erts_smp_atomic_init_rb erts_atomic_init_rb -#define erts_smp_atomic_set_rb erts_atomic_set_rb -#define erts_smp_atomic_read_rb erts_atomic_read_rb -#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb -#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb -#define erts_smp_atomic_inc_rb erts_atomic_inc_rb -#define erts_smp_atomic_dec_rb erts_atomic_dec_rb -#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb -#define erts_smp_atomic_add_rb erts_atomic_add_rb -#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb -#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb -#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb -#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb -#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb - -#define erts_smp_atomic_init_wb erts_atomic_init_wb -#define erts_smp_atomic_set_wb erts_atomic_set_wb -#define erts_smp_atomic_read_wb erts_atomic_read_wb -#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb -#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb -#define erts_smp_atomic_inc_wb erts_atomic_inc_wb -#define erts_smp_atomic_dec_wb erts_atomic_dec_wb -#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb -#define erts_smp_atomic_add_wb erts_atomic_add_wb -#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb -#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb -#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb -#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb -#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb - -#define erts_smp_atomic_set_dirty erts_atomic_set_dirty -#define erts_smp_atomic_read_dirty erts_atomic_read_dirty - -/* 32-bit atomics */ - -#define erts_smp_atomic32_init_nob erts_atomic32_init_nob -#define erts_smp_atomic32_set_nob erts_atomic32_set_nob -#define erts_smp_atomic32_read_nob erts_atomic32_read_nob -#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob -#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob -#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob -#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob -#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob -#define erts_smp_atomic32_add_nob erts_atomic32_add_nob -#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob -#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob -#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob -#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob -#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob - -#define erts_smp_atomic32_init_mb erts_atomic32_init_mb -#define erts_smp_atomic32_set_mb erts_atomic32_set_mb -#define erts_smp_atomic32_read_mb erts_atomic32_read_mb -#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb -#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb -#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb -#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb -#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb -#define erts_smp_atomic32_add_mb erts_atomic32_add_mb -#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb -#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb -#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb -#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb -#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb - -#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb -#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb -#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb -#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb -#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb -#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb -#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb -#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb -#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb -#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb -#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb -#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb -#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb -#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb - -#define erts_smp_atomic32_init_relb erts_atomic32_init_relb -#define erts_smp_atomic32_set_relb erts_atomic32_set_relb -#define erts_smp_atomic32_read_relb erts_atomic32_read_relb -#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb -#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb -#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb -#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb -#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb -#define erts_smp_atomic32_add_relb erts_atomic32_add_relb -#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb -#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb -#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb -#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb -#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb - -#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb -#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb -#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb -#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb -#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb -#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb -#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb -#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb -#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb -#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb -#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb -#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb -#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb -#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb - -#define erts_smp_atomic32_init_rb erts_atomic32_init_rb -#define erts_smp_atomic32_set_rb erts_atomic32_set_rb -#define erts_smp_atomic32_read_rb erts_atomic32_read_rb -#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb -#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb -#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb -#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb -#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb -#define erts_smp_atomic32_add_rb erts_atomic32_add_rb -#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb -#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb -#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb -#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb -#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb - -#define erts_smp_atomic32_init_wb erts_atomic32_init_wb -#define erts_smp_atomic32_set_wb erts_atomic32_set_wb -#define erts_smp_atomic32_read_wb erts_atomic32_read_wb -#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb -#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb -#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb -#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb -#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb -#define erts_smp_atomic32_add_wb erts_atomic32_add_wb -#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb -#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb -#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb -#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb -#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb - -#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty -#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty - -/* 64-bit atomics */ - -#define erts_smp_atomic64_init_nob erts_atomic64_init_nob -#define erts_smp_atomic64_set_nob erts_atomic64_set_nob -#define erts_smp_atomic64_read_nob erts_atomic64_read_nob -#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob -#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob -#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob -#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob -#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob -#define erts_smp_atomic64_add_nob erts_atomic64_add_nob -#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob -#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob -#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob -#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob -#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob - -#define erts_smp_atomic64_init_mb erts_atomic64_init_mb -#define erts_smp_atomic64_set_mb erts_atomic64_set_mb -#define erts_smp_atomic64_read_mb erts_atomic64_read_mb -#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb -#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb -#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb -#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb -#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb -#define erts_smp_atomic64_add_mb erts_atomic64_add_mb -#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb -#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb -#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb -#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb -#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb - -#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb -#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb -#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb -#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb -#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb -#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb -#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb -#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb -#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb -#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb -#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb -#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb -#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb -#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb - -#define erts_smp_atomic64_init_relb erts_atomic64_init_relb -#define erts_smp_atomic64_set_relb erts_atomic64_set_relb -#define erts_smp_atomic64_read_relb erts_atomic64_read_relb -#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb -#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb -#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb -#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb -#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb -#define erts_smp_atomic64_add_relb erts_atomic64_add_relb -#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb -#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb -#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb -#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb -#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb - -#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb -#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb -#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb -#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb -#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb -#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb -#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb -#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb -#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb -#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb -#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb -#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb -#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb -#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb - -#define erts_smp_atomic64_init_rb erts_atomic64_init_rb -#define erts_smp_atomic64_set_rb erts_atomic64_set_rb -#define erts_smp_atomic64_read_rb erts_atomic64_read_rb -#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb -#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb -#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb -#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb -#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb -#define erts_smp_atomic64_add_rb erts_atomic64_add_rb -#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb -#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb -#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb -#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb -#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb - -#define erts_smp_atomic64_init_wb erts_atomic64_init_wb -#define erts_smp_atomic64_set_wb erts_atomic64_set_wb -#define erts_smp_atomic64_read_wb erts_atomic64_read_wb -#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb -#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb -#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb -#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb -#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb -#define erts_smp_atomic64_add_wb erts_atomic64_add_wb -#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb -#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb -#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb -#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb -#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb - -#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty -#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty - -#else /* !ERTS_SMP */ - -/* Double word size atomics */ - -#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set -#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init -#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read -#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg - -#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set -#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read - -/* Word size atomics */ - -#define erts_smp_atomic_init_nob erts_no_atomic_set -#define erts_smp_atomic_set_nob erts_no_atomic_set -#define erts_smp_atomic_read_nob erts_no_atomic_read -#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read -#define erts_smp_atomic_inc_nob erts_no_atomic_inc -#define erts_smp_atomic_dec_nob erts_no_atomic_dec -#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read -#define erts_smp_atomic_add_nob erts_no_atomic_add -#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band -#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset - -#define erts_smp_atomic_init_mb erts_no_atomic_set -#define erts_smp_atomic_set_mb erts_no_atomic_set -#define erts_smp_atomic_read_mb erts_no_atomic_read -#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_mb erts_no_atomic_inc -#define erts_smp_atomic_dec_mb erts_no_atomic_dec -#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read -#define erts_smp_atomic_add_mb erts_no_atomic_add -#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset - -#define erts_smp_atomic_init_acqb erts_no_atomic_set -#define erts_smp_atomic_set_acqb erts_no_atomic_set -#define erts_smp_atomic_read_acqb erts_no_atomic_read -#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_acqb erts_no_atomic_inc -#define erts_smp_atomic_dec_acqb erts_no_atomic_dec -#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read -#define erts_smp_atomic_add_acqb erts_no_atomic_add -#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset - -#define erts_smp_atomic_init_relb erts_no_atomic_set -#define erts_smp_atomic_set_relb erts_no_atomic_set -#define erts_smp_atomic_read_relb erts_no_atomic_read -#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_relb erts_no_atomic_inc -#define erts_smp_atomic_dec_relb erts_no_atomic_dec -#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read -#define erts_smp_atomic_add_relb erts_no_atomic_add -#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset - -#define erts_smp_atomic_init_ddrb erts_no_atomic_set -#define erts_smp_atomic_set_ddrb erts_no_atomic_set -#define erts_smp_atomic_read_ddrb erts_no_atomic_read -#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc -#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec -#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read -#define erts_smp_atomic_add_ddrb erts_no_atomic_add -#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset - -#define erts_smp_atomic_init_rb erts_no_atomic_set -#define erts_smp_atomic_set_rb erts_no_atomic_set -#define erts_smp_atomic_read_rb erts_no_atomic_read -#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_rb erts_no_atomic_inc -#define erts_smp_atomic_dec_rb erts_no_atomic_dec -#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read -#define erts_smp_atomic_add_rb erts_no_atomic_add -#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset - -#define erts_smp_atomic_init_wb erts_no_atomic_set -#define erts_smp_atomic_set_wb erts_no_atomic_set -#define erts_smp_atomic_read_wb erts_no_atomic_read -#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read -#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read -#define erts_smp_atomic_inc_wb erts_no_atomic_inc -#define erts_smp_atomic_dec_wb erts_no_atomic_dec -#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read -#define erts_smp_atomic_add_wb erts_no_atomic_add -#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor -#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band -#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg -#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg -#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset - -#define erts_smp_atomic_set_dirty erts_no_atomic_set -#define erts_smp_atomic_read_dirty erts_no_atomic_read - -/* 32-bit atomics */ - -#define erts_smp_atomic32_init_nob erts_no_atomic32_set -#define erts_smp_atomic32_set_nob erts_no_atomic32_set -#define erts_smp_atomic32_read_nob erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc -#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read -#define erts_smp_atomic32_add_nob erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_mb erts_no_atomic32_set -#define erts_smp_atomic32_set_mb erts_no_atomic32_set -#define erts_smp_atomic32_read_mb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_mb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_acqb erts_no_atomic32_set -#define erts_smp_atomic32_set_acqb erts_no_atomic32_set -#define erts_smp_atomic32_read_acqb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_acqb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_relb erts_no_atomic32_set -#define erts_smp_atomic32_set_relb erts_no_atomic32_set -#define erts_smp_atomic32_read_relb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_relb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set -#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set -#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_rb erts_no_atomic32_set -#define erts_smp_atomic32_set_rb erts_no_atomic32_set -#define erts_smp_atomic32_read_rb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_rb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_init_wb erts_no_atomic32_set -#define erts_smp_atomic32_set_wb erts_no_atomic32_set -#define erts_smp_atomic32_read_wb erts_no_atomic32_read -#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read -#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read -#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc -#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec -#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read -#define erts_smp_atomic32_add_wb erts_no_atomic32_add -#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor -#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band -#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg -#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg -#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset - -#define erts_smp_atomic32_set_dirty erts_no_atomic32_set -#define erts_smp_atomic32_read_dirty erts_no_atomic32_read - -/* 64-bit atomics */ - -#define erts_smp_atomic64_init_nob erts_no_atomic64_set -#define erts_smp_atomic64_set_nob erts_no_atomic64_set -#define erts_smp_atomic64_read_nob erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc -#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read -#define erts_smp_atomic64_add_nob erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_mb erts_no_atomic64_set -#define erts_smp_atomic64_set_mb erts_no_atomic64_set -#define erts_smp_atomic64_read_mb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_mb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_acqb erts_no_atomic64_set -#define erts_smp_atomic64_set_acqb erts_no_atomic64_set -#define erts_smp_atomic64_read_acqb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_acqb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_relb erts_no_atomic64_set -#define erts_smp_atomic64_set_relb erts_no_atomic64_set -#define erts_smp_atomic64_read_relb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_relb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set -#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set -#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_rb erts_no_atomic64_set -#define erts_smp_atomic64_set_rb erts_no_atomic64_set -#define erts_smp_atomic64_read_rb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_rb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_init_wb erts_no_atomic64_set -#define erts_smp_atomic64_set_wb erts_no_atomic64_set -#define erts_smp_atomic64_read_wb erts_no_atomic64_read -#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read -#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read -#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc -#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec -#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read -#define erts_smp_atomic64_add_wb erts_no_atomic64_add -#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor -#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band -#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg -#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg -#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset - -#define erts_smp_atomic64_set_dirty erts_no_atomic64_set -#define erts_smp_atomic64_read_dirty erts_no_atomic64_read - -#endif /* !ERTS_SMP */ - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_smp_thr_init(erts_smp_thr_init_data_t *id) -{ -#ifdef ERTS_SMP - erts_thr_init(id); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg, - erts_smp_thr_opts_t *opts) -{ -#ifdef ERTS_SMP - erts_thr_create(tid, func, arg, opts); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res) -{ -#ifdef ERTS_SMP - erts_thr_join(tid, thr_res); -#endif -} - - -ERTS_GLB_INLINE void -erts_smp_thr_detach(erts_smp_tid_t tid) -{ -#ifdef ERTS_SMP - erts_thr_detach(tid); -#endif -} - - -ERTS_GLB_INLINE void -erts_smp_thr_exit(void *res) -{ -#ifdef ERTS_SMP - erts_thr_exit(res); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_install_exit_handler(void (*exit_handler)(void)) -{ -#ifdef ERTS_SMP - erts_thr_install_exit_handler(exit_handler); -#endif -} - -ERTS_GLB_INLINE erts_smp_tid_t -erts_smp_thr_self(void) -{ -#ifdef ERTS_SMP - return erts_thr_self(); -#else - return 0; -#endif -} - - -ERTS_GLB_INLINE int -erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y) -{ -#ifdef ERTS_SMP - return erts_equal_tids(x, y); -#else - return 1; -#endif -} - - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_rec_mtx_init(mtx); -#endif -} -#endif - -ERTS_GLB_INLINE void -erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) -{ -#ifdef ERTS_SMP - erts_mtx_init_x(mtx, name, extra); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) -{ -#ifdef ERTS_SMP - erts_mtx_init_locked_x_opt(mtx, name, extra, 0); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name) -{ -#ifdef ERTS_SMP - erts_mtx_init(mtx, name); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name) -{ -#ifdef ERTS_SMP - erts_mtx_init_locked(mtx, name); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_destroy(mtx); -#endif -} - -ERTS_GLB_INLINE int -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line) -#else -erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - return erts_mtx_trylock_x(mtx,file,line); -#elif defined(ERTS_SMP) - return erts_mtx_trylock(mtx); -#else - return 0; -#endif - -} - - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line) -#else -erts_smp_mtx_lock(erts_smp_mtx_t *mtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_mtx_lock_x(mtx, file, line); -#elif defined(ERTS_SMP) - erts_mtx_lock(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unlock(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unlock(mtx); -#endif -} - -ERTS_GLB_INLINE int -erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_mtx_is_locked(mtx); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_cnd_init(erts_smp_cnd_t *cnd) -{ -#ifdef ERTS_SMP - erts_cnd_init(cnd); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_cnd_destroy(erts_smp_cnd_t *cnd) -{ -#ifdef ERTS_SMP - erts_cnd_destroy(cnd); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_cnd_wait(cnd, mtx); -#endif -} - -/* - * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast() - * - * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast' - * even though the associated mutex/mutexes isn't/aren't locked by the - * caller. Our implementation do not allow that in order to avoid a - * performance penalty. That is, all associated mutexes *need* to be - * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()! - */ - -ERTS_GLB_INLINE void -erts_smp_cnd_signal(erts_smp_cnd_t *cnd) -{ -#ifdef ERTS_SMP - erts_cnd_signal(cnd); -#endif -} - - -ERTS_GLB_INLINE void -erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) -{ -#ifdef ERTS_SMP - erts_cnd_broadcast(cnd); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_set_reader_group(int no) -{ -#ifdef ERTS_SMP - erts_rwmtx_set_reader_group(no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, - erts_smp_rwmtx_opt_t *opt, - char *name, - Eterm extra) -{ -#ifdef ERTS_SMP - erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) -{ -#ifdef ERTS_SMP - erts_rwmtx_init_x(rwmtx, name, extra); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, - erts_smp_rwmtx_opt_t *opt, - char *name) -{ -#ifdef ERTS_SMP - erts_rwmtx_init_opt(rwmtx, opt, name); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) -{ -#ifdef ERTS_SMP - erts_rwmtx_init(rwmtx, name); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx) -{ -#ifdef ERTS_SMP - erts_rwmtx_destroy(rwmtx); -#endif -} - -ERTS_GLB_INLINE int -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) -#else -erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - return erts_rwmtx_tryrlock_x(rwmtx, file, line); -#elif defined(ERTS_SMP) - return erts_rwmtx_tryrlock(rwmtx); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) -#else -erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_rwmtx_rlock_x(rwmtx, file, line); -#elif defined(ERTS_SMP) - erts_rwmtx_rlock(rwmtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx) -{ -#ifdef ERTS_SMP - erts_rwmtx_runlock(rwmtx); -#endif -} - - -ERTS_GLB_INLINE int -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) -#else -erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - return erts_rwmtx_tryrwlock_x(rwmtx, file, line); -#elif defined(ERTS_SMP) - return erts_rwmtx_tryrwlock(rwmtx); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line) -#else -erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_rwmtx_rwlock_x(rwmtx, file, line); -#elif defined(ERTS_SMP) - erts_rwmtx_rwlock(rwmtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx) -{ -#ifdef ERTS_SMP - erts_rwmtx_rwunlock(rwmtx); -#endif -} - -#if 0 /* The following rwmtx function names are - reserved for potential future use. */ - -/* Try upgrade from r-locked state to rw-locked state */ -ERTS_GLB_INLINE int -erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx) -{ - return 0; -} - -/* Upgrade from r-locked state to rw-locked state */ -ERTS_GLB_INLINE void -erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx) -{ - -} - -/* Downgrade from rw-locked state to r-locked state */ -ERTS_GLB_INLINE void -erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx) -{ - -} - -#endif - -ERTS_GLB_INLINE int -erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_rwmtx_is_rlocked(mtx); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE int -erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_rwmtx_is_rwlocked(mtx); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) -{ -#ifdef ERTS_SMP - erts_spinlock_init_x(lock, name, extra); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name) -{ -#ifdef ERTS_SMP - erts_spinlock_init(lock, name); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock) -{ -#ifdef ERTS_SMP - erts_spinlock_destroy(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_spin_unlock(erts_smp_spinlock_t *lock) -{ -#ifdef ERTS_SMP - erts_spin_unlock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line) -#else -erts_smp_spin_lock(erts_smp_spinlock_t *lock) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_spin_lock_x(lock, file, line); -#elif defined(ERTS_SMP) - erts_spin_lock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE int -erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_spinlock_is_locked(lock); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra) -{ -#ifdef ERTS_SMP - erts_rwlock_init_x(lock, name, extra); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name) -{ -#ifdef ERTS_SMP - erts_rwlock_init(lock, name); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock) -{ -#ifdef ERTS_SMP - erts_rwlock_destroy(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_read_unlock(erts_smp_rwlock_t *lock) -{ -#ifdef ERTS_SMP - erts_read_unlock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line) -#else -erts_smp_read_lock(erts_smp_rwlock_t *lock) -#endif -{ -#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP) - erts_read_lock_x(lock, file, line); -#elif defined(ERTS_SMP) - erts_read_lock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_write_unlock(erts_smp_rwlock_t *lock) -{ -#ifdef ERTS_SMP - erts_write_unlock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -#ifdef ERTS_ENABLE_LOCK_POSITION -erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line) -#else -erts_smp_write_lock(erts_smp_rwlock_t *lock) -#endif -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION) - erts_write_lock_x(lock, file, line); -#elif defined(ERTS_SMP) - erts_write_lock(lock); -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE int -erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_rwlock_is_rlocked(lock); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE int -erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock) -{ -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) - return erts_lc_rwlock_is_rwlocked(lock); -#else - return 0; -#endif -} - -ERTS_GLB_INLINE void -erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname) -{ -#ifdef ERTS_SMP - erts_tsd_key_create(keyp,keyname); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_tsd_key_delete(erts_smp_tsd_key_t key) -{ -#ifdef ERTS_SMP - erts_tsd_key_delete(key); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value) -{ -#ifdef ERTS_SMP - erts_tsd_set(key, value); -#endif -} - -ERTS_GLB_INLINE void * -erts_smp_tsd_get(erts_smp_tsd_key_t key) -{ -#ifdef ERTS_SMP - return erts_tsd_get(key); -#else - return NULL; -#endif -} - -#ifdef ERTS_THR_HAVE_SIG_FUNCS -#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 - -ERTS_GLB_INLINE void -erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset) -{ -#ifdef ERTS_SMP - erts_thr_sigmask(how, set, oset); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_thr_sigwait(const sigset_t *set, int *sig) -{ -#ifdef ERTS_SMP - erts_thr_sigwait(set, sig); -#endif -} - -#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -#endif /* ERL_SMP_H */ - -#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS - -/* Deprecated functions to replace */ - -#undef erts_smp_atomic_init -#undef erts_smp_atomic_set -#undef erts_smp_atomic_read -#undef erts_smp_atomic_inctest -#undef erts_smp_atomic_dectest -#undef erts_smp_atomic_inc -#undef erts_smp_atomic_dec -#undef erts_smp_atomic_addtest -#undef erts_smp_atomic_add -#undef erts_smp_atomic_xchg -#undef erts_smp_atomic_cmpxchg -#undef erts_smp_atomic_bor -#undef erts_smp_atomic_band - -#undef erts_smp_atomic32_init -#undef erts_smp_atomic32_set -#undef erts_smp_atomic32_read -#undef erts_smp_atomic32_inctest -#undef erts_smp_atomic32_dectest -#undef erts_smp_atomic32_inc -#undef erts_smp_atomic32_dec -#undef erts_smp_atomic32_addtest -#undef erts_smp_atomic32_add -#undef erts_smp_atomic32_xchg -#undef erts_smp_atomic32_cmpxchg -#undef erts_smp_atomic32_bor -#undef erts_smp_atomic32_band - -#endif diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 842802f8d9..6daf043117 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -270,7 +270,6 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define is_byte(x) (((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL) #define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL) #define is_not_valid_bit_size(x) (!is_valid_bit_size((x))) -#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2) #define _unchecked_unsigned_val(x) ((x) >> _TAG_IMMED1_SIZE) _ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm) #define unsigned_val(x) _ET_APPLY(unsigned_val,(x)) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 700ed90def..96824dc06e 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -80,7 +80,6 @@ #include "erl_thr_progress.h" #include "global.h" -#ifdef ERTS_SMP #define ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE 0 @@ -321,13 +320,23 @@ tmp_thr_prgr_data(ErtsSchedulerData *esdp) ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); if (!tpd) { - /* - * We only allocate the part up to the wakeup_request field - * which is the first field only used by registered threads - */ - tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, - offsetof(ErtsThrPrgrData, wakeup_request)); - init_tmp_thr_prgr_data(tpd); + /* + * We only allocate the part up to the wakeup_request field which is + * the first field only used by registered threads + */ + size_t alloc_size = offsetof(ErtsThrPrgrData, wakeup_request); + + /* We may land here as a result of unmanaged_delay being called from + * the lock counting module, which in turn might be called from within + * the allocator, so we use plain malloc to avoid deadlocks. */ + tpd = +#ifdef ERTS_ENABLE_LOCK_COUNT + malloc(alloc_size); +#else + erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, alloc_size); +#endif + + init_tmp_thr_prgr_data(tpd); } return tpd; @@ -337,8 +346,13 @@ static ERTS_INLINE void return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) { if (tpd->is_temporary) { - erts_tsd_set(erts_thr_prgr_data_key__, NULL); - erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); + erts_tsd_set(erts_thr_prgr_data_key__, NULL); + +#ifdef ERTS_ENABLE_LOCK_COUNT + free(tpd); +#else + erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); +#endif } } @@ -1498,4 +1512,3 @@ void erts_thr_progress_dbg_print_state(void) } -#endif diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index b2894ba1fe..fa936b5707 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -33,18 +33,6 @@ #include "sys.h" -#ifndef ERTS_SMP - -#define erts_smp_thr_progress_block() ((void) 0) -#define erts_smp_thr_progress_unblock() ((void) 0) -#define erts_smp_thr_progress_is_blocking() 1 - -#else /* ERTS_SMP */ - -#define erts_smp_thr_progress_block erts_thr_progress_block -#define erts_smp_thr_progress_unblock erts_thr_progress_unblock -#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking - void erts_thr_progress_block(void); void erts_thr_progress_unblock(void); int erts_thr_progress_is_blocking(void); @@ -87,13 +75,10 @@ typedef struct { int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp); void erts_thr_progress_fatal_error_wait(SWord timeout); -#endif /* ERTS_SMP */ typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp; struct ErtsThrPrgrLaterOp_ { -#ifdef ERTS_SMP ErtsThrPrgrVal later; -#endif void (*func)(void *); void *data; ErtsThrPrgrLaterOp *next; @@ -107,7 +92,6 @@ struct ErtsThrPrgrLaterOp_ { #include "erl_threads.h" #include "erl_process.h" -#ifdef ERTS_SMP /* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */ #define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0) @@ -324,6 +308,5 @@ erts_thr_progress_has_reached(ErtsThrPrgrVal val) #endif -#endif /* ERTS_SMP */ #endif diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index f56d0828dd..548c2768e5 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -87,32 +87,10 @@ #define ERTS_THR_Q_MAX_FINI_DEQ_OPS 50 -#ifdef ERTS_SMP ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(sl_element, ErtsThrQElement_t, 1000, ERTS_ALC_T_THR_Q_EL_SL) -#else - -static void -init_sl_element_alloc(void) -{ -} - -static ErtsThrQElement_t * -sl_element_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_THR_Q_EL_SL, - sizeof(ErtsThrQElement_t)); -} - -static void -sl_element_free(ErtsThrQElement_t *p) -{ - erts_free(ERTS_ALC_T_THR_Q_EL_SL, p); -} - -#endif #define ErtsThrQDirtyReadEl(A) \ ((ErtsThrQElement_t *) erts_atomic_read_dirty((A))) @@ -135,14 +113,6 @@ static void noop_callback(void *arg) { } void erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) { -#ifndef USE_THREADS - q->init = *qi; - if (!q->init.notify) - q->init.notify = noop_callback; - q->first = NULL; - q->last = NULL; - q->q.blk = NULL; -#else erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL); q->tail.data.marker.data.ptr = NULL; erts_atomic_init_nob(&q->tail.data.last, @@ -164,10 +134,8 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) q->head.deq_fini.automatic = qi->auto_finalize_dequeue; q->head.deq_fini.start = NULL; q->head.deq_fini.end = NULL; -#ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_current(); q->head.next.thr_progress_reached = 1; -#endif q->head.next.um_refc_ix = 1; q->head.next.unref_end = &q->tail.data.marker; q->head.used_marker = 1; @@ -176,15 +144,12 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi) q->q.finalizing = 0; q->q.live = qi->live.queue; q->q.blk = NULL; -#endif } ErtsThrQCleanState_t erts_thr_q_finalize(ErtsThrQ_t *q) { -#ifdef USE_THREADS q->q.finalizing = 1; -#endif while (erts_thr_q_dequeue(q)); return erts_thr_q_clean(q); } @@ -229,7 +194,6 @@ erts_thr_q_destroy(ErtsThrQ_t *q) return erts_thr_q_finalize(q); } -#ifdef USE_THREADS static void destroy(ErtsThrQ_t *q) @@ -249,7 +213,6 @@ destroy(ErtsThrQ_t *q) erts_free(atype, q->q.blk); } -#endif static ERTS_INLINE ErtsThrQElement_t * element_live_alloc(ErtsThrQLive_t live) @@ -267,11 +230,7 @@ static ERTS_INLINE ErtsThrQElement_t * element_alloc(ErtsThrQ_t *q) { ErtsThrQLive_t live; -#ifdef USE_THREADS live = q->tail.data.live; -#else - live = q->init.live.objects; -#endif return element_live_alloc(live); } @@ -291,15 +250,10 @@ static ERTS_INLINE void element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el) { ErtsThrQLive_t live; -#ifdef USE_THREADS live = q->head.live; -#else - live = q->init.live.objects; -#endif element_live_free(live, el); } -#ifdef USE_THREADS static ERTS_INLINE ErtsThrQElement_t * enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this) @@ -423,11 +377,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) return ERTS_THR_Q_CLEAN; } -#ifdef ERTS_SMP if (q->head.next.thr_progress_reached || erts_thr_progress_has_reached(q->head.next.thr_progress)) { q->head.next.thr_progress_reached = 1; -#endif um_refc_ix = q->head.next.um_refc_ix; if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) { /* Move unreferenced end pointer forward... */ @@ -439,23 +391,17 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) ilast = (erts_aint_t) enqueue_marker(q, NULL); if (q->head.unref_end == (ErtsThrQElement_t *) ilast) - ERTS_SMP_MEMORY_BARRIER; + ERTS_THR_MEMORY_BARRIER; else { q->head.next.unref_end = (ErtsThrQElement_t *) ilast; -#ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_later(NULL); -#endif erts_atomic32_set_relb(&q->tail.data.um_refc_ix, um_refc_ix); q->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0; -#ifdef ERTS_SMP q->head.next.thr_progress_reached = 0; -#endif } } -#ifdef ERTS_SMP } -#endif head = ErtsThrQDirtyReadEl(&q->head.head); if (q->head.first == head) { @@ -489,9 +435,7 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) check_thr_progress: -#ifdef ERTS_SMP if (q->head.next.thr_progress_reached) -#endif { int um_refc_ix = q->head.next.um_refc_ix; if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) { @@ -505,24 +449,16 @@ check_thr_progress: return ERTS_THR_Q_NEED_THR_PRGR; } -#endif ErtsThrQCleanState_t erts_thr_q_clean(ErtsThrQ_t *q) { -#ifdef USE_THREADS return clean(q, ERTS_THR_Q_MAX_SCHED_CLEAN_OPS, 0); -#else - return ERTS_THR_Q_CLEAN; -#endif } ErtsThrQCleanState_t erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty) { -#ifndef USE_THREADS - return ERTS_THR_Q_CLEAN; -#else ErtsThrQElement_t *head = ErtsThrQDirtyReadEl(&q->head.head); if (ensure_empty) { erts_aint_t inext; @@ -553,39 +489,21 @@ erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty) if (q->head.first != q->head.unref_end) return ERTS_THR_Q_DIRTY; -#ifdef ERTS_SMP if (q->head.next.thr_progress_reached) -#endif { int um_refc_ix = q->head.next.um_refc_ix; if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) return ERTS_THR_Q_DIRTY; } return ERTS_THR_Q_NEED_THR_PRGR; -#endif } static void enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) { -#ifndef USE_THREADS - ASSERT(data); - - this->next = NULL; - this->data.ptr = data; - - if (q->last) - q->last->next = this; - else { - q->first = q->last = this; - q->init.notify(q->init.arg); - } -#else int notify; int um_refc_ix = 0; -#ifdef ERTS_SMP int unmanaged_thread; -#endif #if ERTS_THR_Q_DBG_CHK_DATA if (!data) @@ -596,10 +514,8 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) this->data.ptr = data; -#ifdef ERTS_SMP unmanaged_thread = !erts_thr_progress_is_managed_thread(); if (unmanaged_thread) -#endif { um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix); while (1) { @@ -616,9 +532,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) notify = this == enqueue_managed(q, this); -#ifdef ERTS_SMP if (unmanaged_thread) -#endif { if (notify) erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]); @@ -627,7 +541,6 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) } if (notify) q->tail.data.notify(q->tail.data.arg); -#endif } void @@ -645,9 +558,6 @@ erts_thr_q_prepare_enqueue(ErtsThrQ_t *q) int erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp) { -#ifndef USE_THREADS - return 0; -#else #ifdef DEBUG if (!q->head.deq_fini.start) { ASSERT(!q->head.deq_fini.end); @@ -670,14 +580,12 @@ erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp) q->head.deq_fini.start = NULL; q->head.deq_fini.end = NULL; return fdp->start != NULL; -#endif } void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0, ErtsThrQFinDeQ_t *fdp1) { -#ifdef USE_THREADS if (fdp1->start) { if (fdp0->end) ErtsThrQDirtySetEl(&fdp0->end->next, fdp1->start); @@ -685,13 +593,11 @@ erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0, fdp0->start = fdp1->start; fdp0->end = fdp1->end; } -#endif } int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state) { -#ifdef USE_THREADS ErtsThrQElement_t *start = state->start; if (start) { ErtsThrQLive_t live; @@ -710,17 +616,14 @@ int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state) return 1; /* More to do */ state->end = NULL; } -#endif return 0; } void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *state) { -#ifdef USE_THREADS state->start = NULL; state->end = NULL; -#endif } @@ -734,22 +637,6 @@ erts_thr_q_enqueue_prepared(ErtsThrQ_t *q, void *data, ErtsThrQPrepEnQ_t *prep) void * erts_thr_q_dequeue(ErtsThrQ_t *q) { -#ifndef USE_THREADS - void *res; - ErtsThrQElement_t *tmp; - - if (!q->first) - return NULL; - tmp = q->first; - res = tmp->data.ptr; - q->first = tmp->next; - if (!q->first) - q->last = NULL; - - element_free(q, tmp); - - return res; -#else ErtsThrQElement_t *head; erts_aint_t inext; void *res; @@ -778,7 +665,6 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) ? ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS : ERTS_THR_Q_MAX_SCHED_CLEAN_OPS), 1); return res; -#endif } #ifdef USE_LTTNG_VM_TRACEPOINTS @@ -786,14 +672,6 @@ int erts_thr_q_length_dirty(ErtsThrQ_t *q) { int n = 0; -#ifndef USE_THREADS - void *res; - ErtsThrQElement_t *tmp; - - for (tmp = q->first; tmp != NULL; tmp = tmp->next) { - n++; - } -#else ErtsThrQElement_t *e; erts_aint_t inext; @@ -808,7 +686,6 @@ erts_thr_q_length_dirty(ErtsThrQ_t *q) } inext = erts_atomic_read_acqb(&e->next); } -#endif return n; } #endif diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h index 705a67af4c..163a25318d 100644 --- a/erts/emulator/beam/erl_thr_queue.h +++ b/erts/emulator/beam/erl_thr_queue.h @@ -78,11 +78,7 @@ typedef struct ErtsThrQElement_t_ ErtsThrQElement_t; typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t; struct ErtsThrQElement_t_ { -#ifdef USE_THREADS erts_atomic_t next; -#else - ErtsThrQElement_t *next; -#endif union { erts_atomic_t atmc; void *ptr; @@ -100,7 +96,6 @@ typedef enum { ERTS_THR_Q_DIRTY, } ErtsThrQCleanState_t; -#ifdef USE_THREADS typedef struct { ErtsThrQElement_t marker; @@ -108,9 +103,7 @@ typedef struct { erts_atomic_t um_refc[2]; erts_atomic32_t um_refc_ix; ErtsThrQLive_t live; -#ifdef ERTS_SMP erts_atomic32_t thr_prgr_clean_scheduled; -#endif void *arg; void (*notify)(void *); } ErtsThrQTail_t; @@ -141,10 +134,8 @@ struct ErtsThrQ_t_ { ErtsThrQElement_t *end; } deq_fini; struct { -#ifdef ERTS_SMP ErtsThrPrgrVal thr_progress; int thr_progress_reached; -#endif int um_refc_ix; ErtsThrQElement_t *unref_end; } next; @@ -159,18 +150,6 @@ struct ErtsThrQ_t_ { } q; }; -#else /* !USE_THREADS */ - -struct ErtsThrQ_t_ { - ErtsThrQInit_t init; - ErtsThrQElement_t *first; - ErtsThrQElement_t *last; - struct { - void *blk; - } q; -}; - -#endif void erts_thr_q_init(void); void erts_thr_q_initialize(ErtsThrQ_t *, ErtsThrQInit_t *); @@ -194,19 +173,15 @@ void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *); int erts_thr_q_length_dirty(ErtsThrQ_t *); #endif -#ifdef ERTS_SMP ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q); -#endif #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ERTS_SMP ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q) { return q->head.next.thr_progress; } -#endif #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 9612b70469..aedceb6fc2 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -45,11 +45,6 @@ * Data dependency read barrier. Orders *only* loads * according to data dependency across the barrier. * - * If thread support has been disabled, these barriers will become no-ops. - * - * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will - * be enabled only in the SMP enabled runtime system. - * * --- Atomic operations --- * * Atomics operations exist for 32-bit, word size, and double word size @@ -86,20 +81,6 @@ * barrier. Load in atomic operation is ordered * before the barrier. * - * If thread support has been disabled, these functions are mapped to - * functions that performs the same operation, but aren't atomic - * and don't imply any memory barriers. - * - * If the atomic operations are prefixed with erts_smp_ instead of only - * erts_ the atomic operations will only be atomic in the SMP enabled - * runtime system, and will be mapped to non-atomic operations without - * memory barriers in the runtime system without SMP support. Atomic - * operations with erts_smp_ prefix should use the atomic types - * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t - * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The - * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t - * are the same. - * * --- 32-bit atomic operations --- * * The following 32-bit atomic operations exist. <B> should be @@ -259,13 +240,15 @@ #include "sys.h" -#ifdef USE_THREADS +#include "erl_lock_flags.h" +#include "erl_term.h" + #define ETHR_TRY_INLINE_FUNCS #include "ethread.h" + #include "erl_lock_check.h" #include "erl_lock_count.h" -#include "erl_term.h" #if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4 /* @@ -307,9 +290,11 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; +#endif +#ifdef DEBUG + erts_lock_flags_t flags; #endif - } erts_mtx_t; typedef ethr_cond erts_cnd_t; @@ -320,7 +305,10 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; +#endif +#ifdef DEBUG + erts_lock_flags_t flags; #endif } erts_rwmtx_t; @@ -365,7 +353,10 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; +#endif +#ifdef DEBUG + erts_lock_flags_t flags; #endif } erts_spinlock_t; @@ -376,7 +367,10 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; +#endif +#ifdef DEBUG + erts_lock_flags_t flags; #endif } erts_rwlock_t; @@ -391,76 +385,6 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif -#else /* #ifdef USE_THREADS */ - -#define ERTS_THR_MEMORY_BARRIER -#define ERTS_THR_WRITE_MEMORY_BARRIER -#define ERTS_THR_READ_MEMORY_BARRIER -#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER - -#define ERTS_THR_OPTS_DEFAULT_INITER 0 -typedef int erts_thr_opts_t; -typedef int erts_thr_init_data_t; -typedef int erts_thr_late_init_data_t; -typedef int erts_tid_t; -typedef int erts_mtx_t; -typedef int erts_cnd_t; -#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} -#define ERTS_RWMTX_TYPE_NORMAL 0 -#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 -#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 -#define ERTS_RWMTX_LONG_LIVED 0 -#define ERTS_RWMTX_SHORT_LIVED 0 -#define ERTS_RWMTX_UNKNOWN_LIVED 0 -typedef struct { - char type; - char lived; - int main_spincount; - int aux_spincount; -} erts_rwmtx_opt_t; -typedef int erts_rwmtx_t; -typedef int erts_tsd_key_t; -typedef int erts_tse_t; - -typedef struct { SWord sint[2]; } erts_dw_aint_t; -typedef SWord erts_aint_t; -typedef Sint32 erts_aint32_t; -typedef Sint64 erts_aint64_t; - -#define erts_dw_atomic_t erts_dw_aint_t -#define erts_atomic_t erts_aint_t -#define erts_atomic32_t erts_aint32_t -#define erts_atomic64_t erts_aint64_t - -#if __GNUC__ > 2 -typedef struct { } erts_spinlock_t; -typedef struct { } erts_rwlock_t; -#else -typedef struct { int gcc_is_buggy; } erts_spinlock_t; -typedef struct { int gcc_is_buggy; } erts_rwlock_t; -#endif - -#ifdef WORDS_BIGENDIAN -#define ERTS_DW_AINT_LOW_WORD 1 -#define ERTS_DW_AINT_HIGH_WORD 0 -#else -#define ERTS_DW_AINT_LOW_WORD 0 -#define ERTS_DW_AINT_HIGH_WORD 1 -#endif - -#define ERTS_MTX_INITER 0 -#define ERTS_CND_INITER 0 -#define ERTS_THR_INIT_DATA_DEF_INITER 0 - -#define ERTS_HAVE_REC_MTX_INIT 1 - -#endif /* #ifdef USE_THREADS */ - -#define erts_no_dw_atomic_t erts_dw_aint_t -#define erts_no_atomic_t erts_aint_t -#define erts_no_atomic32_t erts_aint32_t -#define erts_no_atomic64_t erts_aint64_t - #define ERTS_AINT_NULL ((erts_aint_t) NULL) #define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) @@ -479,11 +403,14 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); -ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); -ERTS_GLB_INLINE void erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); -ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); -ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); +ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, + char *name, + Eterm extra, + erts_lock_flags_t flags); +ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, + char *name, + Eterm extra, + erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, @@ -502,18 +429,15 @@ ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); -ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, - erts_rwmtx_opt_t *opt, - char *name, - Eterm extra); -ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, - char *name, - Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, - erts_rwmtx_opt_t *opt, - char *name); + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra, + erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, - char *name); + char *name, + Eterm extra, + erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); #ifdef ERTS_ENABLE_LOCK_POSITION ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line); @@ -530,89 +454,10 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); - -ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); -ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val); -ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, - erts_no_dw_atomic_t *val, - erts_no_dw_atomic_t *old_val); -ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp); -ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp); -ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp, - erts_aint_t i); -ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var, - erts_aint_t mask); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp, - erts_aint_t new); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected); -ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var, - erts_aint_t mask, - erts_aint_t set); -ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var, - erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp); -ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp); -ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp, - erts_aint32_t i); -ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp, - erts_aint32_t i); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var, - erts_aint32_t mask); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, - erts_aint32_t new); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected); -ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, - erts_aint32_t mask, - erts_aint32_t set); -ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var, - erts_aint64_t i); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp); -ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp); -ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp, - erts_aint64_t i); -ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp, - erts_aint64_t i); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var, - erts_aint64_t mask); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var, - erts_aint64_t mask); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, - erts_aint64_t new); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, - erts_aint64_t new, - erts_aint64_t expected); -ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var, - erts_aint64_t mask, - erts_aint64_t set); - -ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, - char *name, - Eterm extra, - Uint16 opt); -ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, - char *name, - Eterm extra); ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, - char *name); + char *name, + Eterm extra, + erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock); ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock); #ifdef ERTS_ENABLE_LOCK_POSITION @@ -621,11 +466,10 @@ ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigne ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock); #endif ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock); -ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock, - char *name, - Eterm extra); ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock, - char *name); + char *name, + Eterm extra, + erts_lock_flags_t flags); ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock); #ifdef ERTS_ENABLE_LOCK_POSITION @@ -663,13 +507,10 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset); ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); -#ifdef USE_THREADS ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig); -#endif #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ -#ifdef USE_THREADS ERTS_GLB_INLINE erts_aint_t erts_atomic_read_bset_nob(erts_atomic_t *var, @@ -1677,379 +1518,6 @@ erts_atomic64_read_dirty(erts_atomic64_t *var) #endif /* ARCH_32 */ -#else /* !USE_THREADS */ - -/* Double word size atomics */ - -#define erts_dw_atomic_init_nob erts_no_dw_atomic_set -#define erts_dw_atomic_set_nob erts_no_dw_atomic_set -#define erts_dw_atomic_read_nob erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_mb erts_no_dw_atomic_init -#define erts_dw_atomic_set_mb erts_no_dw_atomic_set -#define erts_dw_atomic_read_mb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init -#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set -#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_relb erts_no_dw_atomic_init -#define erts_dw_atomic_set_relb erts_no_dw_atomic_set -#define erts_dw_atomic_read_relb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init -#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set -#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_rb erts_no_dw_atomic_init -#define erts_dw_atomic_set_rb erts_no_dw_atomic_set -#define erts_dw_atomic_read_rb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_init_wb erts_no_dw_atomic_init -#define erts_dw_atomic_set_wb erts_no_dw_atomic_set -#define erts_dw_atomic_read_wb erts_no_dw_atomic_read -#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg - -#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set -#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read - -/* Word size atomics */ - -#define erts_atomic_init_nob erts_no_atomic_set -#define erts_atomic_set_nob erts_no_atomic_set -#define erts_atomic_read_nob erts_no_atomic_read -#define erts_atomic_inc_read_nob erts_no_atomic_inc_read -#define erts_atomic_dec_read_nob erts_no_atomic_dec_read -#define erts_atomic_inc_nob erts_no_atomic_inc -#define erts_atomic_dec_nob erts_no_atomic_dec -#define erts_atomic_add_read_nob erts_no_atomic_add_read -#define erts_atomic_add_nob erts_no_atomic_add -#define erts_atomic_read_bor_nob erts_no_atomic_read_bor -#define erts_atomic_read_band_nob erts_no_atomic_read_band -#define erts_atomic_xchg_nob erts_no_atomic_xchg -#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_nob erts_no_atomic_read_bset - -#define erts_atomic_init_mb erts_no_atomic_set -#define erts_atomic_set_mb erts_no_atomic_set -#define erts_atomic_read_mb erts_no_atomic_read -#define erts_atomic_inc_read_mb erts_no_atomic_inc_read -#define erts_atomic_dec_read_mb erts_no_atomic_dec_read -#define erts_atomic_inc_mb erts_no_atomic_inc -#define erts_atomic_dec_mb erts_no_atomic_dec -#define erts_atomic_add_read_mb erts_no_atomic_add_read -#define erts_atomic_add_mb erts_no_atomic_add -#define erts_atomic_read_bor_mb erts_no_atomic_read_bor -#define erts_atomic_read_band_mb erts_no_atomic_read_band -#define erts_atomic_xchg_mb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_mb erts_no_atomic_read_bset - -#define erts_atomic_init_acqb erts_no_atomic_set -#define erts_atomic_set_acqb erts_no_atomic_set -#define erts_atomic_read_acqb erts_no_atomic_read -#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read -#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read -#define erts_atomic_inc_acqb erts_no_atomic_inc -#define erts_atomic_dec_acqb erts_no_atomic_dec -#define erts_atomic_add_read_acqb erts_no_atomic_add_read -#define erts_atomic_add_acqb erts_no_atomic_add -#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor -#define erts_atomic_read_band_acqb erts_no_atomic_read_band -#define erts_atomic_xchg_acqb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset - -#define erts_atomic_init_relb erts_no_atomic_set -#define erts_atomic_set_relb erts_no_atomic_set -#define erts_atomic_read_relb erts_no_atomic_read -#define erts_atomic_inc_read_relb erts_no_atomic_inc_read -#define erts_atomic_dec_read_relb erts_no_atomic_dec_read -#define erts_atomic_inc_relb erts_no_atomic_inc -#define erts_atomic_dec_relb erts_no_atomic_dec -#define erts_atomic_add_read_relb erts_no_atomic_add_read -#define erts_atomic_add_relb erts_no_atomic_add -#define erts_atomic_read_bor_relb erts_no_atomic_read_bor -#define erts_atomic_read_band_relb erts_no_atomic_read_band -#define erts_atomic_xchg_relb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_relb erts_no_atomic_read_bset - -#define erts_atomic_init_ddrb erts_no_atomic_set -#define erts_atomic_set_ddrb erts_no_atomic_set -#define erts_atomic_read_ddrb erts_no_atomic_read -#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read -#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read -#define erts_atomic_inc_ddrb erts_no_atomic_inc -#define erts_atomic_dec_ddrb erts_no_atomic_dec -#define erts_atomic_add_read_ddrb erts_no_atomic_add_read -#define erts_atomic_add_ddrb erts_no_atomic_add -#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor -#define erts_atomic_read_band_ddrb erts_no_atomic_read_band -#define erts_atomic_xchg_ddrb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset - -#define erts_atomic_init_rb erts_no_atomic_set -#define erts_atomic_set_rb erts_no_atomic_set -#define erts_atomic_read_rb erts_no_atomic_read -#define erts_atomic_inc_read_rb erts_no_atomic_inc_read -#define erts_atomic_dec_read_rb erts_no_atomic_dec_read -#define erts_atomic_inc_rb erts_no_atomic_inc -#define erts_atomic_dec_rb erts_no_atomic_dec -#define erts_atomic_add_read_rb erts_no_atomic_add_read -#define erts_atomic_add_rb erts_no_atomic_add -#define erts_atomic_read_bor_rb erts_no_atomic_read_bor -#define erts_atomic_read_band_rb erts_no_atomic_read_band -#define erts_atomic_xchg_rb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_rb erts_no_atomic_read_bset - -#define erts_atomic_init_wb erts_no_atomic_set -#define erts_atomic_set_wb erts_no_atomic_set -#define erts_atomic_read_wb erts_no_atomic_read -#define erts_atomic_inc_read_wb erts_no_atomic_inc_read -#define erts_atomic_dec_read_wb erts_no_atomic_dec_read -#define erts_atomic_inc_wb erts_no_atomic_inc -#define erts_atomic_dec_wb erts_no_atomic_dec -#define erts_atomic_add_read_wb erts_no_atomic_add_read -#define erts_atomic_add_wb erts_no_atomic_add -#define erts_atomic_read_bor_wb erts_no_atomic_read_bor -#define erts_atomic_read_band_wb erts_no_atomic_read_band -#define erts_atomic_xchg_wb erts_no_atomic_xchg -#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg -#define erts_atomic_read_bset_wb erts_no_atomic_read_bset - -#define erts_atomic_set_dirty erts_no_atomic_set -#define erts_atomic_read_dirty erts_no_atomic_read - -/* 32-bit atomics */ - -#define erts_atomic32_init_nob erts_no_atomic32_set -#define erts_atomic32_set_nob erts_no_atomic32_set -#define erts_atomic32_read_nob erts_no_atomic32_read -#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read -#define erts_atomic32_inc_nob erts_no_atomic32_inc -#define erts_atomic32_dec_nob erts_no_atomic32_dec -#define erts_atomic32_add_read_nob erts_no_atomic32_add_read -#define erts_atomic32_add_nob erts_no_atomic32_add -#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor -#define erts_atomic32_read_band_nob erts_no_atomic32_read_band -#define erts_atomic32_xchg_nob erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset - -#define erts_atomic32_init_mb erts_no_atomic32_set -#define erts_atomic32_set_mb erts_no_atomic32_set -#define erts_atomic32_read_mb erts_no_atomic32_read -#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read -#define erts_atomic32_inc_mb erts_no_atomic32_inc -#define erts_atomic32_dec_mb erts_no_atomic32_dec -#define erts_atomic32_add_read_mb erts_no_atomic32_add_read -#define erts_atomic32_add_mb erts_no_atomic32_add -#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_mb erts_no_atomic32_read_band -#define erts_atomic32_xchg_mb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset - -#define erts_atomic32_init_acqb erts_no_atomic32_set -#define erts_atomic32_set_acqb erts_no_atomic32_set -#define erts_atomic32_read_acqb erts_no_atomic32_read -#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read -#define erts_atomic32_inc_acqb erts_no_atomic32_inc -#define erts_atomic32_dec_acqb erts_no_atomic32_dec -#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read -#define erts_atomic32_add_acqb erts_no_atomic32_add -#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band -#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset - -#define erts_atomic32_init_relb erts_no_atomic32_set -#define erts_atomic32_set_relb erts_no_atomic32_set -#define erts_atomic32_read_relb erts_no_atomic32_read -#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read -#define erts_atomic32_inc_relb erts_no_atomic32_inc -#define erts_atomic32_dec_relb erts_no_atomic32_dec -#define erts_atomic32_add_read_relb erts_no_atomic32_add_read -#define erts_atomic32_add_relb erts_no_atomic32_add -#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_relb erts_no_atomic32_read_band -#define erts_atomic32_xchg_relb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset - -#define erts_atomic32_init_ddrb erts_no_atomic32_set -#define erts_atomic32_set_ddrb erts_no_atomic32_set -#define erts_atomic32_read_ddrb erts_no_atomic32_read -#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read -#define erts_atomic32_inc_ddrb erts_no_atomic32_inc -#define erts_atomic32_dec_ddrb erts_no_atomic32_dec -#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read -#define erts_atomic32_add_ddrb erts_no_atomic32_add -#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band -#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset - -#define erts_atomic32_init_rb erts_no_atomic32_set -#define erts_atomic32_set_rb erts_no_atomic32_set -#define erts_atomic32_read_rb erts_no_atomic32_read -#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read -#define erts_atomic32_inc_rb erts_no_atomic32_inc -#define erts_atomic32_dec_rb erts_no_atomic32_dec -#define erts_atomic32_add_read_rb erts_no_atomic32_add_read -#define erts_atomic32_add_rb erts_no_atomic32_add -#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_rb erts_no_atomic32_read_band -#define erts_atomic32_xchg_rb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset - -#define erts_atomic32_init_wb erts_no_atomic32_set -#define erts_atomic32_set_wb erts_no_atomic32_set -#define erts_atomic32_read_wb erts_no_atomic32_read -#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read -#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read -#define erts_atomic32_inc_wb erts_no_atomic32_inc -#define erts_atomic32_dec_wb erts_no_atomic32_dec -#define erts_atomic32_add_read_wb erts_no_atomic32_add_read -#define erts_atomic32_add_wb erts_no_atomic32_add -#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor -#define erts_atomic32_read_band_wb erts_no_atomic32_read_band -#define erts_atomic32_xchg_wb erts_no_atomic32_xchg -#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg -#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset - -#define erts_atomic32_set_dirty erts_no_atomic32_set -#define erts_atomic32_read_dirty erts_no_atomic32_read - -/* 64-bit atomics */ - -#define erts_atomic64_init_nob erts_no_atomic64_set -#define erts_atomic64_set_nob erts_no_atomic64_set -#define erts_atomic64_read_nob erts_no_atomic64_read -#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read -#define erts_atomic64_inc_nob erts_no_atomic64_inc -#define erts_atomic64_dec_nob erts_no_atomic64_dec -#define erts_atomic64_add_read_nob erts_no_atomic64_add_read -#define erts_atomic64_add_nob erts_no_atomic64_add -#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor -#define erts_atomic64_read_band_nob erts_no_atomic64_read_band -#define erts_atomic64_xchg_nob erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset - -#define erts_atomic64_init_mb erts_no_atomic64_set -#define erts_atomic64_set_mb erts_no_atomic64_set -#define erts_atomic64_read_mb erts_no_atomic64_read -#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read -#define erts_atomic64_inc_mb erts_no_atomic64_inc -#define erts_atomic64_dec_mb erts_no_atomic64_dec -#define erts_atomic64_add_read_mb erts_no_atomic64_add_read -#define erts_atomic64_add_mb erts_no_atomic64_add -#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_mb erts_no_atomic64_read_band -#define erts_atomic64_xchg_mb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset - -#define erts_atomic64_init_acqb erts_no_atomic64_set -#define erts_atomic64_set_acqb erts_no_atomic64_set -#define erts_atomic64_read_acqb erts_no_atomic64_read -#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read -#define erts_atomic64_inc_acqb erts_no_atomic64_inc -#define erts_atomic64_dec_acqb erts_no_atomic64_dec -#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read -#define erts_atomic64_add_acqb erts_no_atomic64_add -#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band -#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset - -#define erts_atomic64_init_relb erts_no_atomic64_set -#define erts_atomic64_set_relb erts_no_atomic64_set -#define erts_atomic64_read_relb erts_no_atomic64_read -#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read -#define erts_atomic64_inc_relb erts_no_atomic64_inc -#define erts_atomic64_dec_relb erts_no_atomic64_dec -#define erts_atomic64_add_read_relb erts_no_atomic64_add_read -#define erts_atomic64_add_relb erts_no_atomic64_add -#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_relb erts_no_atomic64_read_band -#define erts_atomic64_xchg_relb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset - -#define erts_atomic64_init_ddrb erts_no_atomic64_set -#define erts_atomic64_set_ddrb erts_no_atomic64_set -#define erts_atomic64_read_ddrb erts_no_atomic64_read -#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read -#define erts_atomic64_inc_ddrb erts_no_atomic64_inc -#define erts_atomic64_dec_ddrb erts_no_atomic64_dec -#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read -#define erts_atomic64_add_ddrb erts_no_atomic64_add -#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band -#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset - -#define erts_atomic64_init_rb erts_no_atomic64_set -#define erts_atomic64_set_rb erts_no_atomic64_set -#define erts_atomic64_read_rb erts_no_atomic64_read -#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read -#define erts_atomic64_inc_rb erts_no_atomic64_inc -#define erts_atomic64_dec_rb erts_no_atomic64_dec -#define erts_atomic64_add_read_rb erts_no_atomic64_add_read -#define erts_atomic64_add_rb erts_no_atomic64_add -#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_rb erts_no_atomic64_read_band -#define erts_atomic64_xchg_rb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset - -#define erts_atomic64_init_wb erts_no_atomic64_set -#define erts_atomic64_set_wb erts_no_atomic64_set -#define erts_atomic64_read_wb erts_no_atomic64_read -#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read -#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read -#define erts_atomic64_inc_wb erts_no_atomic64_inc -#define erts_atomic64_dec_wb erts_no_atomic64_dec -#define erts_atomic64_add_read_wb erts_no_atomic64_add_read -#define erts_atomic64_add_wb erts_no_atomic64_add -#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor -#define erts_atomic64_read_band_wb erts_no_atomic64_read_band -#define erts_atomic64_xchg_wb erts_no_atomic64_xchg -#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg -#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset - -#define erts_atomic64_set_dirty erts_no_atomic64_set -#define erts_atomic64_read_dirty erts_no_atomic64_read - -#endif /* !USE_THREADS */ #include "erl_msacc.h" @@ -2058,211 +1526,127 @@ erts_atomic64_read_dirty(erts_atomic64_t *var) ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id) { -#ifdef USE_THREADS int res = ethr_init(id); if (res) erts_thr_fatal_error(res, "initialize thread library"); -#endif } ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id) { -#ifdef USE_THREADS int res = ethr_late_init(id); if (res) erts_thr_fatal_error(res, "complete initialization of thread library"); -#endif } ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { -#ifdef USE_THREADS int res = ethr_thr_create(tid, func, arg, opts); if (res) erts_thr_fatal_error(res, "create thread"); -#endif } ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res) { -#ifdef USE_THREADS int res = ethr_thr_join(tid, thr_res); if (res) erts_thr_fatal_error(res, "join thread"); -#endif } ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid) { -#ifdef USE_THREADS int res = ethr_thr_detach(tid); if (res) erts_thr_fatal_error(res, "detach thread"); -#endif } ERTS_GLB_INLINE void erts_thr_exit(void *res) { -#ifdef USE_THREADS ethr_thr_exit(res); erts_thr_fatal_error(0, "terminate thread"); -#endif } ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)) { -#ifdef USE_THREADS int res = ethr_install_exit_handler(exit_handler); if (res != 0) erts_thr_fatal_error(res, "install thread exit handler"); -#endif } ERTS_GLB_INLINE erts_tid_t erts_thr_self(void) { -#ifdef USE_THREADS return ethr_self(); -#else - return 0; -#endif } ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len) { -#ifdef USE_THREADS return ethr_getname(tid, buf, len); -#else - return -1; -#endif } ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y) { -#ifdef USE_THREADS return ethr_equal_tids(x, y); -#else - return 1; -#endif } ERTS_GLB_INLINE void -erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) +erts_mtx_init(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags) { -#ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize mutex"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); -#endif -#endif -} + if (res) { + erts_thr_fatal_error(res, "initialize mutex"); + } -ERTS_GLB_INLINE void -erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) -{ -#ifdef USE_THREADS - int res = ethr_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize mutex"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + flags |= ERTS_LOCK_TYPE_MUTEX; +#ifdef DEBUG + mtx->flags = flags; #endif -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) -{ -#ifdef USE_THREADS - int res = ethr_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize mutex"); #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); + erts_lc_init_lock_x(&mtx->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); -#endif - ethr_mutex_lock(&mtx->mtx); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_trylock(1, &mtx->lc); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock(&mtx->lcnt, 1); -#endif + erts_lcnt_init_ref_x(&mtx->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void -erts_mtx_init(erts_mtx_t *mtx, char *name) +erts_mtx_init_locked(erts_mtx_t *mtx, char *name, Eterm extra, erts_lock_flags_t flags) { -#ifdef USE_THREADS - int res = ethr_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize mutex"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); -#endif -#endif -} + erts_mtx_init(mtx, name, extra, flags); -ERTS_GLB_INLINE void -erts_mtx_init_locked(erts_mtx_t *mtx, char *name) -{ -#ifdef USE_THREADS - int res = ethr_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize mutex"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); -#endif ethr_mutex_lock(&mtx->mtx); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_trylock(1, &mtx->lc); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock(&mtx->lcnt, 1); -#endif -#endif + #ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_trylock(1, &mtx->lc); + #endif + #ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock(&mtx->lcnt, 1); + #endif } ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx) { -#ifdef USE_THREADS int res; + + ASSERT(!(mtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&mtx->lcnt); + erts_lcnt_uninstall(&mtx->lcnt); #endif res = ethr_mutex_destroy(&mtx->mtx); if (res != 0) { @@ -2276,7 +1660,6 @@ erts_mtx_destroy(erts_mtx_t *mtx) #endif erts_thr_fatal_error(res, "destroy mutex"); } -#endif } ERTS_GLB_INLINE int @@ -2286,7 +1669,6 @@ erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line) erts_mtx_trylock(erts_mtx_t *mtx) #endif { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -2308,9 +1690,6 @@ erts_mtx_trylock(erts_mtx_t *mtx) erts_lcnt_trylock(&mtx->lcnt, res); #endif return res; -#else - return 0; -#endif } @@ -2321,7 +1700,6 @@ erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line) erts_mtx_lock(erts_mtx_t *mtx) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_x(&mtx->lc, file, line); @@ -2336,13 +1714,11 @@ erts_mtx_lock(erts_mtx_t *mtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif -#endif } ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif @@ -2350,16 +1726,16 @@ erts_mtx_unlock(erts_mtx_t *mtx) erts_lcnt_unlock(&mtx->lcnt); #endif ethr_mutex_unlock(&mtx->mtx); -#endif } ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; - lc.flags = 0; + lc.flags = ERTS_LOCK_FLAGS_TYPE_MUTEX; + lc.taken_options = 0; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -2370,17 +1746,14 @@ erts_lc_mtx_is_locked(erts_mtx_t *mtx) ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd) { -#ifdef USE_THREADS int res = ethr_cond_init(cnd); if (res) erts_thr_fatal_error(res, "initialize condition variable"); -#endif } ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd) { -#ifdef USE_THREADS int res = ethr_cond_destroy(cnd); if (res != 0) { #ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG @@ -2393,13 +1766,11 @@ erts_cnd_destroy(erts_cnd_t *cnd) #endif erts_thr_fatal_error(res, "destroy condition variable"); } -#endif } ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) { -#ifdef USE_THREADS int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -2421,7 +1792,6 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) if (res != 0 && res != EINTR) erts_thr_fatal_error(res, "wait on condition variable"); ERTS_MSACC_POP_STATE(); -#endif } /* @@ -2437,18 +1807,14 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx) ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { -#ifdef USE_THREADS ethr_cond_signal(cnd); -#endif } ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { -#ifdef USE_THREADS ethr_cond_broadcast(cnd); -#endif } /* rwmutex */ @@ -2456,81 +1822,54 @@ erts_cnd_broadcast(erts_cnd_t *cnd) ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no) { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); + erts_lc_check_no_locked_of_type(ERTS_LOCK_TYPE_RWMUTEX); #endif res = ethr_rwmutex_set_reader_group(no); if (res != 0) erts_thr_fatal_error(res, "set reader group"); -#endif } ERTS_GLB_INLINE void -erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, - erts_rwmtx_opt_t *opt, - char *name, - Eterm extra) -{ -#ifdef USE_THREADS +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_rwmtx_opt_t *opt, + char *name, Eterm extra, erts_lock_flags_t flags) { int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); - if (res != 0) - erts_thr_fatal_error(res, "initialize rwmutex"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - if (name && name[0] == '\0') - erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra); - else - erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra); -#endif -#endif -} + if (res != 0) { + erts_thr_fatal_error(res, "initialize rwmutex"); + } -ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, - char *name, - Eterm extra) -{ - erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); -} + flags |= ERTS_LOCK_TYPE_RWMUTEX; +#ifdef DEBUG + rwmtx->flags = flags; +#endif -ERTS_GLB_INLINE void -erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, - erts_rwmtx_opt_t *opt, - char *name) -{ -#ifdef USE_THREADS - int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); - if (res != 0) - erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX); + erts_lc_init_lock_x(&rwmtx->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX); -#endif + erts_lcnt_init_ref_x(&rwmtx->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) -{ - erts_rwmtx_init_opt(rwmtx, NULL, name); +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra, + erts_lock_flags_t flags) { + erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags); } ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { -#ifdef USE_THREADS int res; + + ASSERT(!(rwmtx->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&rwmtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&rwmtx->lcnt); + erts_lcnt_uninstall(&rwmtx->lcnt); #endif res = ethr_rwmutex_destroy(&rwmtx->rwmtx); if (res != 0) { @@ -2544,7 +1883,6 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) #endif erts_thr_fatal_error(res, "destroy rwmutex"); } -#endif } ERTS_GLB_INLINE int @@ -2554,11 +1892,10 @@ erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #endif { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_CHECK - if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ)) + if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ)) return EBUSY; /* Make sure caller can handle the situation without causing a lock order violation */ #endif @@ -2567,19 +1904,16 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line); + erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else - erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ); + erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); + erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_READ); #endif return res; -#else - return 0; -#endif } ERTS_GLB_INLINE void @@ -2589,36 +1923,32 @@ erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line); + erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else - erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); + erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif -#endif } ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); + erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_rwmutex_runlock(&rwmtx->rwmtx); -#endif } @@ -2629,11 +1959,10 @@ erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #endif { -#ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_CHECK - if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE)) + if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR)) return EBUSY; /* Make sure caller can handle the situation without causing a lock order violation */ #endif @@ -2642,19 +1971,16 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); + erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else - erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); + erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LOCK_OPTIONS_RDWR); #endif return res; -#else - return 0; -#endif } ERTS_GLB_INLINE void @@ -2664,36 +1990,32 @@ erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line) erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); + erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else - erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); + erts_lc_lock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif -#endif } ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); + erts_lc_unlock_flg(&rwmtx->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_rwmutex_rwunlock(&rwmtx->rwmtx); -#endif } #if 0 /* The following rwmtx function names are @@ -2725,10 +2047,11 @@ erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx) ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; - lc.flags = ERTS_LC_FLG_LO_READ; + lc.flags = ERTS_LOCK_TYPE_RWMUTEX; + lc.taken_options = ERTS_LOCK_OPTIONS_READ; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -2739,10 +2062,11 @@ erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx) ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = mtx->lc; - lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE; + lc.flags = ERTS_LOCK_TYPE_RWMUTEX; + lc.taken_options = ERTS_LOCK_OPTIONS_RDWR; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -2750,396 +2074,41 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) #endif } -/* No atomic ops */ - -ERTS_GLB_INLINE void -erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) -{ - var->sint[0] = val->sint[0]; - var->sint[1] = val->sint[1]; -} - -ERTS_GLB_INLINE void -erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val) -{ - val->sint[0] = var->sint[0]; - val->sint[1] = var->sint[1]; -} - -ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var, - erts_no_dw_atomic_t *new_val, - erts_no_dw_atomic_t *old_val) -{ - if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) { - erts_no_dw_atomic_read(var, old_val); - return 0; - } - else { - erts_no_dw_atomic_set(var, new_val); - return !0; - } -} - -ERTS_GLB_INLINE void -erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i) -{ - *var = i; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_read(erts_no_atomic_t *var) -{ - return *var; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_inc_read(erts_no_atomic_t *incp) -{ - return ++(*incp); -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_dec_read(erts_no_atomic_t *decp) -{ - return --(*decp); -} - -ERTS_GLB_INLINE void -erts_no_atomic_inc(erts_no_atomic_t *incp) -{ - ++(*incp); -} - -ERTS_GLB_INLINE void -erts_no_atomic_dec(erts_no_atomic_t *decp) -{ - --(*decp); -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i) -{ - return *addp += i; -} - -ERTS_GLB_INLINE void -erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i) -{ - *addp += i; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask) -{ - erts_aint_t old; - old = *var; - *var |= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask) -{ - erts_aint_t old; - old = *var; - *var &= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new) -{ - erts_aint_t old = *xchgp; - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t expected) -{ - erts_aint_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint_t -erts_no_atomic_read_bset(erts_no_atomic_t *var, - erts_aint_t mask, - erts_aint_t set) -{ - erts_aint_t old = *var; - *var &= ~mask; - *var |= (mask & set); - return old; -} - -/* atomic32 */ - -ERTS_GLB_INLINE void -erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i) -{ - *var = i; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_read(erts_no_atomic32_t *var) -{ - return *var; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_inc_read(erts_no_atomic32_t *incp) -{ - return ++(*incp); -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_dec_read(erts_no_atomic32_t *decp) -{ - return --(*decp); -} - -ERTS_GLB_INLINE void -erts_no_atomic32_inc(erts_no_atomic32_t *incp) -{ - ++(*incp); -} - -ERTS_GLB_INLINE void -erts_no_atomic32_dec(erts_no_atomic32_t *decp) -{ - --(*decp); -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i) -{ - return *addp += i; -} - -ERTS_GLB_INLINE void -erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i) -{ - *addp += i; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask) -{ - erts_aint32_t old; - old = *var; - *var |= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask) -{ - erts_aint32_t old; - old = *var; - *var &= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new) -{ - erts_aint32_t old = *xchgp; - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp, - erts_aint32_t new, - erts_aint32_t expected) -{ - erts_aint32_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint32_t -erts_no_atomic32_read_bset(erts_no_atomic32_t *var, - erts_aint32_t mask, - erts_aint32_t set) -{ - erts_aint32_t old = *var; - *var &= ~mask; - *var |= (mask & set); - return old; -} - -/* atomic64 */ - -ERTS_GLB_INLINE void -erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i) -{ - *var = i; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_read(erts_no_atomic64_t *var) -{ - return *var; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_inc_read(erts_no_atomic64_t *incp) -{ - return ++(*incp); -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_dec_read(erts_no_atomic64_t *decp) -{ - return --(*decp); -} - -ERTS_GLB_INLINE void -erts_no_atomic64_inc(erts_no_atomic64_t *incp) -{ - ++(*incp); -} - -ERTS_GLB_INLINE void -erts_no_atomic64_dec(erts_no_atomic64_t *decp) -{ - --(*decp); -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i) -{ - return *addp += i; -} - -ERTS_GLB_INLINE void -erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i) -{ - *addp += i; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask) -{ - erts_aint64_t old; - old = *var; - *var |= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask) -{ - erts_aint64_t old; - old = *var; - *var &= mask; - return old; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new) -{ - erts_aint64_t old = *xchgp; - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, - erts_aint64_t new, - erts_aint64_t expected) -{ - erts_aint64_t old = *xchgp; - if (old == expected) - *xchgp = new; - return old; -} - -ERTS_GLB_INLINE erts_aint64_t -erts_no_atomic64_read_bset(erts_no_atomic64_t *var, - erts_aint64_t mask, - erts_aint64_t set) -{ - erts_aint64_t old = *var; - *var &= ~mask; - *var |= (mask & set); - return old; -} - /* spinlock */ ERTS_GLB_INLINE void -erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) +erts_spinlock_init(erts_spinlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags) { -#ifdef USE_THREADS int res = ethr_spinlock_init(&lock->slck); - if (res) - erts_thr_fatal_error(res, "init spinlock"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra); -#endif -#else - (void)lock; -#endif -} + if (res) { + erts_thr_fatal_error(res, "init spinlock"); + } -ERTS_GLB_INLINE void -erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, - Uint16 opt) -{ -#ifdef USE_THREADS - int res = ethr_spinlock_init(&lock->slck); - if (res) - erts_thr_fatal_error(res, "init spinlock"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); + flags |= ERTS_LOCK_TYPE_SPINLOCK; +#ifdef DEBUG + lock->flags = flags; #endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); -#endif -#else - (void)lock; -#endif -} - -ERTS_GLB_INLINE void -erts_spinlock_init(erts_spinlock_t *lock, char *name) -{ -#ifdef USE_THREADS - int res = ethr_spinlock_init(&lock->slck); - if (res) - erts_thr_fatal_error(res, "init spinlock"); #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK); + erts_lc_init_lock_x(&lock->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK); -#endif -#else - (void)lock; + erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock) { -#ifdef USE_THREADS int res; + + ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&lock->lcnt); + erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_spinlock_destroy(&lock->slck); if (res != 0) { @@ -3153,15 +2122,11 @@ erts_spinlock_destroy(erts_spinlock_t *lock) #endif erts_thr_fatal_error(res, "destroy rwlock"); } -#else - (void)lock; -#endif } ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif @@ -3169,9 +2134,6 @@ erts_spin_unlock(erts_spinlock_t *lock) erts_lcnt_unlock(&lock->lcnt); #endif ethr_spin_unlock(&lock->slck); -#else - (void)lock; -#endif } ERTS_GLB_INLINE void @@ -3181,7 +2143,6 @@ erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line) erts_spin_lock(erts_spinlock_t *lock) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION erts_lc_lock_x(&lock->lc,file,line); @@ -3196,18 +2157,16 @@ erts_spin_lock(erts_spinlock_t *lock) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif -#else - (void)lock; -#endif } ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; - lc.flags = 0; + lc.flags = ERTS_LOCK_TYPE_SPINLOCK; + lc.taken_options = 0; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -3218,51 +2177,38 @@ erts_lc_spinlock_is_locked(erts_spinlock_t *lock) /* rwspinlock */ ERTS_GLB_INLINE void -erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra) +erts_rwlock_init(erts_rwlock_t *lock, char *name, Eterm extra, erts_lock_flags_t flags) { -#ifdef USE_THREADS int res = ethr_rwlock_init(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "init rwlock"); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra); -#endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra); -#endif -#else - (void)lock; + if (res) { + erts_thr_fatal_error(res, "init rwlock"); + } + + flags |= ERTS_LOCK_TYPE_RWSPINLOCK; +#ifdef DEBUG + lock->flags = flags; #endif -} -ERTS_GLB_INLINE void -erts_rwlock_init(erts_rwlock_t *lock, char *name) -{ -#ifdef USE_THREADS - int res = ethr_rwlock_init(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "init rwlock"); #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK); + erts_lc_init_lock_x(&lock->lc, name, flags, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK); -#endif -#else - (void)lock; + erts_lcnt_init_ref_x(&lock->lcnt, name, extra, flags); #endif } ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock) { -#ifdef USE_THREADS int res; + + ASSERT(!(lock->flags & ERTS_LOCK_FLAGS_PROPERTY_STATIC)); + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&lock->lcnt); + erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_rwlock_destroy(&lock->rwlck); if (res != 0) { @@ -3276,25 +2222,18 @@ erts_rwlock_destroy(erts_rwlock_t *lock) #endif erts_thr_fatal_error(res, "destroy rwlock"); } -#else - (void)lock; -#endif } ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); + erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_read_unlock(&lock->rwlck); -#else - (void)lock; -#endif } ERTS_GLB_INLINE void @@ -3304,40 +2243,32 @@ erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) erts_read_lock(erts_rwlock_t *lock) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line); + erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_READ,file,line); #else - erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); + erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_READ); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); + erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_READ); #endif ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif -#else - (void)lock; -#endif } ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); + erts_lc_unlock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_write_unlock(&lock->rwlck); -#else - (void)lock; -#endif } ERTS_GLB_INLINE void @@ -3347,33 +2278,30 @@ erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line) erts_write_lock(erts_rwlock_t *lock) #endif { -#ifdef USE_THREADS #ifdef ERTS_ENABLE_LOCK_CHECK #ifdef ERTS_ENABLE_LOCK_POSITION - erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line); + erts_lc_lock_flg_x(&lock->lc, ERTS_LOCK_OPTIONS_RDWR,file,line); #else - erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); + erts_lc_lock_flg(&lock->lc, ERTS_LOCK_OPTIONS_RDWR); #endif #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); + erts_lcnt_lock_opt(&lock->lcnt, ERTS_LOCK_OPTIONS_RDWR); #endif ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif -#else - (void)lock; -#endif } ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; - lc.flags = ERTS_LC_FLG_LO_READ; + lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK; + lc.taken_options = ERTS_LOCK_OPTIONS_READ; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -3384,10 +2312,11 @@ erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock) ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) { -#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) int res; erts_lc_lock_t lc = lock->lc; - lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE; + lc.flags = ERTS_LOCK_TYPE_RWSPINLOCK; + lc.taken_options = ERTS_LOCK_OPTIONS_RDWR; erts_lc_have_locks(&res, &lc, 1); return res; #else @@ -3398,125 +2327,90 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname) { -#ifdef USE_THREADS int res = ethr_tsd_key_create(keyp, keyname); if (res) erts_thr_fatal_error(res, "create thread specific data key"); -#endif } ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key) { -#ifdef USE_THREADS int res = ethr_tsd_key_delete(key); if (res) erts_thr_fatal_error(res, "delete thread specific data key"); -#endif } ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value) { -#ifdef USE_THREADS int res = ethr_tsd_set(key, value); if (res) erts_thr_fatal_error(res, "set thread specific data"); -#endif } ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key) { -#ifdef USE_THREADS return ethr_tsd_get(key); -#else - return NULL; -#endif } ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { -#ifdef USE_THREADS return (erts_tse_t *) ethr_get_ts_event(); -#else - return (erts_tse_t *) NULL; -#endif } ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { -#ifdef USE_THREADS ethr_leave_ts_event(ep); -#endif } ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep) { -#ifdef USE_THREADS int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event); if (res != 0) erts_thr_fatal_error(res, "prepare timed"); -#endif } ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { -#ifdef USE_THREADS ethr_event_set(&((ethr_ts_event *) ep)->event); -#endif } ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { -#ifdef USE_THREADS ethr_event_reset(&((ethr_ts_event *) ep)->event); -#endif } ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) { -#ifdef USE_THREADS int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_wait(&((ethr_ts_event *) ep)->event); ERTS_MSACC_POP_STATE(); return res; -#else - return ENOTSUP; -#endif } ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) { -#ifdef USE_THREADS int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); ERTS_MSACC_POP_STATE(); return res; -#else - return ENOTSUP; -#endif } ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo) { -#ifdef USE_THREADS int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_twait(&((ethr_ts_event *) ep)->event, (ethr_sint64_t) tmo); ERTS_MSACC_POP_STATE(); return res; -#else - return ENOTSUP; -#endif } ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) { -#ifdef USE_THREADS int res; ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_SLEEP); res = ethr_event_stwait(&((ethr_ts_event *) ep)->event, @@ -3524,49 +2418,34 @@ ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo) (ethr_sint64_t) tmo); ERTS_MSACC_POP_STATE(); return res; -#else - return ENOTSUP; -#endif } ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { -#ifdef USE_THREADS return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; -#else - return 0; -#endif } ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) { -#ifdef USE_THREADS int res = ethr_set_main_thr_status(on, no); if (res != 0) erts_thr_fatal_error(res, "set thread main status"); -#endif } ERTS_GLB_INLINE int erts_thr_get_main_status(void) { -#ifdef USE_THREADS int main_status; int res = ethr_get_main_thr_status(&main_status); if (res != 0) erts_thr_fatal_error(res, "get thread main status"); return main_status; -#else - return 1; -#endif } ERTS_GLB_INLINE void erts_thr_yield(void) { -#ifdef USE_THREADS int res = ETHR_YIELD(); if (res != 0) erts_thr_fatal_error(res, "yield"); -#endif } @@ -3574,34 +2453,28 @@ ERTS_GLB_INLINE void erts_thr_yield(void) ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig) { -#ifdef USE_THREADS int res = ethr_kill((ethr_tid)tid, sig); if (res) erts_thr_fatal_error(res, "killing thread"); -#endif } ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset) { -#ifdef USE_THREADS int res = ethr_sigmask(how, set, oset); if (res) erts_thr_fatal_error(res, "get or set signal mask"); -#endif } ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig) { -#ifdef USE_THREADS int res; do { res = ethr_sigwait(set, sig); } while (res == EINTR); if (res) erts_thr_fatal_error(res, "to wait for signal"); -#endif } #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ @@ -3609,37 +2482,3 @@ erts_thr_sigwait(const sigset_t *set, int *sig) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifndef ERL_THREAD_H__ */ - -#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS - -/* Deprecated functions to replace */ - -#undef erts_atomic_init -#undef erts_atomic_set -#undef erts_atomic_read -#undef erts_atomic_inctest -#undef erts_atomic_dectest -#undef erts_atomic_inc -#undef erts_atomic_dec -#undef erts_atomic_addtest -#undef erts_atomic_add -#undef erts_atomic_xchg -#undef erts_atomic_cmpxchg -#undef erts_atomic_bor -#undef erts_atomic_band - -#undef erts_atomic32_init -#undef erts_atomic32_set -#undef erts_atomic32_read -#undef erts_atomic32_inctest -#undef erts_atomic32_dectest -#undef erts_atomic32_inc -#undef erts_atomic32_dec -#undef erts_atomic32_addtest -#undef erts_atomic32_add -#undef erts_atomic32_xchg -#undef erts_atomic32_cmpxchg -#undef erts_atomic32_bor -#undef erts_atomic32_band - -#endif diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index ccc5526664..27164d50a0 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -130,6 +130,13 @@ Eterm erts_get_monotonic_end_time(struct process *c_p); Eterm erts_monotonic_time_source(struct process*c_p); Eterm erts_system_time_source(struct process*c_p); +void erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user, + ErtsMonotonicTime *ms_sys, + ErtsMonotonicTime *ms_user_diff, + ErtsMonotonicTime *ms_sys_diff); +void erts_wall_clock_elapsed_both(ErtsMonotonicTime *total, + ErtsMonotonicTime *diff); + #ifdef SYS_CLOCK_RESOLUTION #define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000)) #else diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 3084a8db75..f2e0900fec 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -36,12 +36,29 @@ #include "erl_driver.h" #include "erl_nif.h" -static erts_smp_mtx_t erts_timeofday_mtx; -static erts_smp_mtx_t erts_get_time_mtx; +static erts_mtx_t erts_get_time_mtx; -static SysTimes t_start; /* Used in elapsed_time_both */ -static ErtsMonotonicTime prev_wall_clock_elapsed; /* Used in wall_clock_elapsed_time_both */ -static ErtsMonotonicTime previous_now; /* Used in get_now */ + /* used by erts_runtime_elapsed_both */ +typedef struct { + erts_mtx_t mtx; + ErtsMonotonicTime user; + ErtsMonotonicTime sys; +} ErtsRunTimePrevData; + +static union { + ErtsRunTimePrevData data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunTimePrevData))]; +} runtime_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static union { + erts_atomic64_t time; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_atomic64_t))]; +} wall_clock_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +static union { + erts_atomic64_t time; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_atomic64_t))]; +} now_prev erts_align_attribute(ERTS_CACHE_LINE_SIZE); static ErtsMonitor *time_offset_monitors = NULL; static Uint no_time_offset_monitors = 0; @@ -140,7 +157,7 @@ typedef struct { struct time_sup_infrequently_changed__ { #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT struct { - erts_smp_rwmtx_t rwmtx; + erts_rwmtx_t rwmtx; ErtsTWheelTimer timer; ErtsMonotonicCorrectionData cdata; } parmon; @@ -148,9 +165,9 @@ struct time_sup_infrequently_changed__ { #endif ErtsSystemTime sinit; ErtsMonotonicTime not_corrected_moffset; - erts_smp_atomic64_t offset; + erts_atomic64_t offset; ErtsMonotonicTime shadow_offset; - erts_smp_atomic32_t preliminary_offset; + erts_atomic32_t preliminary_offset; }; struct time_sup_frequently_changed__ { @@ -188,19 +205,19 @@ erts_get_approx_time(void) static ERTS_INLINE void init_time_offset(ErtsMonotonicTime offset) { - erts_smp_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset); + erts_atomic64_init_nob(&time_sup.inf.c.offset, (erts_aint64_t) offset); } static ERTS_INLINE void set_time_offset(ErtsMonotonicTime offset) { - erts_smp_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset); + erts_atomic64_set_relb(&time_sup.inf.c.offset, (erts_aint64_t) offset); } static ERTS_INLINE ErtsMonotonicTime get_time_offset(void) { - return (ErtsMonotonicTime) erts_smp_atomic64_read_acqb(&time_sup.inf.c.offset); + return (ErtsMonotonicTime) erts_atomic64_read_acqb(&time_sup.inf.c.offset); } static ERTS_INLINE void @@ -281,7 +298,7 @@ read_corrected_time(int os_drift_corrected) ErtsMonotonicTime os_mtime; ErtsMonotonicCorrectionInstance ci; - erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); os_mtime = erts_os_monotonic_time(); @@ -294,7 +311,7 @@ read_corrected_time(int os_drift_corrected) ci = time_sup.inf.c.parmon.cdata.insts.prev; } - erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); return calc_corrected_erl_mtime(os_mtime, &ci, NULL, os_drift_corrected); @@ -372,13 +389,13 @@ check_time_correction(void *vesdp) int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; int set_new_correction = 0, begin_short_intervals = 0; - erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); erts_os_times(&os_mtime, &os_stime); ci = time_sup.inf.c.parmon.cdata.insts.curr; - erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) erts_exit(ERTS_ABORT_EXIT, @@ -393,7 +410,7 @@ check_time_correction(void *vesdp) if (time_sup.inf.c.shadow_offset) { ERTS_TIME_ASSERT(time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE); - if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) sdiff += time_sup.inf.c.shadow_offset; else time_sup.inf.c.shadow_offset = 0; @@ -416,7 +433,7 @@ check_time_correction(void *vesdp) } } else if ((time_sup.r.o.warp_mode == ERTS_SINGLE_TIME_WARP_MODE - && erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + && erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) && (sdiff < -2*time_sup.r.o.adj.small_diff || 2*time_sup.r.o.adj.small_diff < sdiff)) { /* @@ -641,7 +658,7 @@ check_time_correction(void *vesdp) #endif if (set_new_correction) { - erts_smp_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_rwlock(&time_sup.inf.c.parmon.rwmtx); os_mtime = erts_os_monotonic_time(); @@ -669,7 +686,7 @@ check_time_correction(void *vesdp) time_sup.inf.c.parmon.cdata.insts.curr.os_mtime = os_mtime; time_sup.inf.c.parmon.cdata.insts.curr.correction = new_correction; - erts_smp_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_rwunlock(&time_sup.inf.c.parmon.rwmtx); } if (!esdp) @@ -787,13 +804,13 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep) ErtsMonotonicCorrectionInstance ci; int os_drift_corrected = time_sup.r.o.os_corrected_monotonic_time; - erts_smp_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_rlock(&time_sup.inf.c.parmon.rwmtx); erts_os_times(&os_mtime, stimep); ci = time_sup.inf.c.parmon.cdata.insts.curr; - erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); + erts_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) erts_exit(ERTS_ABORT_EXIT, @@ -846,7 +863,7 @@ static ErtsMonotonicTime get_not_corrected_time(void) { ErtsMonotonicTime stime, mtime; - erts_smp_mtx_lock(&erts_get_time_mtx); + erts_mtx_lock(&erts_get_time_mtx); stime = erts_os_system_time(); @@ -872,7 +889,7 @@ static ErtsMonotonicTime get_not_corrected_time(void) ASSERT(stime == mtime + time_sup.inf.c.not_corrected_moffset); - erts_smp_mtx_unlock(&erts_get_time_mtx); + erts_mtx_unlock(&erts_get_time_mtx); return mtime; } @@ -954,16 +971,20 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) ASSERT(ERTS_MONOTONIC_TIME_MIN < ERTS_MONOTONIC_TIME_MAX); - erts_smp_mtx_init(&erts_timeofday_mtx, "timeofday"); - erts_smp_mtx_init(&erts_get_time_mtx, "get_time"); + erts_mtx_init(&erts_get_time_mtx, "get_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + erts_mtx_init(&runtime_prev.data.mtx, "runtime", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + runtime_prev.data.user = 0; + runtime_prev.data.sys = 0; time_sup.r.o.correction = time_correction; time_sup.r.o.warp_mode = time_warp_mode; if (time_warp_mode == ERTS_SINGLE_TIME_WARP_MODE) - erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1); + erts_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 1); else - erts_smp_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0); + erts_atomic32_init_nob(&time_sup.inf.c.preliminary_offset, 0); time_sup.inf.c.shadow_offset = 0; #if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT @@ -1107,7 +1128,7 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) if (time_sup.r.o.correction) { ErtsMonotonicCorrectionData *cdatap; - erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER; ErtsMonotonicTime offset; erts_os_times(&time_sup.inf.c.minit, &time_sup.inf.c.sinit); @@ -1117,11 +1138,12 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) offset -= ERTS_MONOTONIC_BEGIN; init_time_offset(offset); - rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, - &rwmtx_opts, "get_corrected_time"); + erts_rwmtx_init_opt(&time_sup.inf.c.parmon.rwmtx, &rwmtx_opts, + "get_corrected_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); cdatap = &time_sup.inf.c.parmon.cdata; @@ -1154,9 +1176,13 @@ erts_init_time_sup(int time_correction, ErtsTimeWarpMode time_warp_mode) time_sup.f.c.last_not_corrected_time = 0; } - prev_wall_clock_elapsed = 0; + erts_atomic64_init_nob(&wall_clock_prev.time, + (erts_aint64_t) 0); + + erts_atomic64_init_nob( + &now_prev.time, + (erts_aint64_t) ERTS_MONOTONIC_TO_USEC(get_time_offset())); - previous_now = ERTS_MONOTONIC_TO_USEC(get_time_offset()); #ifdef DEBUG time_sup_initialized = 1; @@ -1197,7 +1223,7 @@ ErtsTimeOffsetState erts_time_offset_state(void) case ERTS_NO_TIME_WARP_MODE: return ERTS_TIME_OFFSET_FINAL; case ERTS_SINGLE_TIME_WARP_MODE: - if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) + if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) return ERTS_TIME_OFFSET_PRELIMINARY; return ERTS_TIME_OFFSET_FINAL; case ERTS_MULTI_TIME_WARP_MODE: @@ -1230,9 +1256,9 @@ erts_finalize_time_offset(void) case ERTS_SINGLE_TIME_WARP_MODE: { ErtsTimeOffsetState res = ERTS_TIME_OFFSET_FINAL; - erts_smp_mtx_lock(&erts_get_time_mtx); + erts_mtx_lock(&erts_get_time_mtx); - if (erts_smp_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) { + if (erts_atomic32_read_nob(&time_sup.inf.c.preliminary_offset)) { ErtsMonotonicTime mtime, new_offset; #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT @@ -1269,11 +1295,11 @@ erts_finalize_time_offset(void) set_time_offset(new_offset); schedule_send_time_offset_changed_notifications(new_offset); - erts_smp_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0); + erts_atomic32_set_nob(&time_sup.inf.c.preliminary_offset, 0); res = ERTS_TIME_OFFSET_PRELIMINARY; } - erts_smp_mtx_unlock(&erts_get_time_mtx); + erts_mtx_unlock(&erts_get_time_mtx); return res; } @@ -1286,56 +1312,93 @@ erts_finalize_time_offset(void) /* info functions */ void -elapsed_time_both(UWord *ms_user, UWord *ms_sys, - UWord *ms_user_diff, UWord *ms_sys_diff) +erts_runtime_elapsed_both(ErtsMonotonicTime *ms_user, ErtsMonotonicTime *ms_sys, + ErtsMonotonicTime *ms_user_diff, ErtsMonotonicTime *ms_sys_diff) { - UWord prev_total_user, prev_total_sys; - UWord total_user, total_sys; + ErtsMonotonicTime prev_user, prev_sys, user, sys; + +#ifdef HAVE_GETRUSAGE + + struct rusage now; + + if (getrusage(RUSAGE_SELF, &now) != 0) { + erts_exit(ERTS_ABORT_EXIT, "getrusage(RUSAGE_SELF, _) failed: %d\n", errno); + return; + } + + user = (ErtsMonotonicTime) now.ru_utime.tv_sec; + user *= (ErtsMonotonicTime) 1000000; + user += (ErtsMonotonicTime) now.ru_utime.tv_usec; + user /= (ErtsMonotonicTime) 1000; + + sys = (ErtsMonotonicTime) now.ru_stime.tv_sec; + sys *= (ErtsMonotonicTime) 1000000; + sys += (ErtsMonotonicTime) now.ru_stime.tv_usec; + sys /= (ErtsMonotonicTime) 1000; + +#else + SysTimes now; sys_times(&now); - total_user = (now.tms_utime * 1000) / SYS_CLK_TCK; - total_sys = (now.tms_stime * 1000) / SYS_CLK_TCK; + user = (ErtsMonotonicTime) now.tms_utime; + user *= (ErtsMonotonicTime) 1000; + user /= (ErtsMonotonicTime) SYS_CLK_TCK; - if (ms_user != NULL) - *ms_user = total_user; - if (ms_sys != NULL) - *ms_sys = total_sys; + sys = (ErtsMonotonicTime) now.tms_stime; + sys *= (ErtsMonotonicTime) 1000; + sys /= (ErtsMonotonicTime) SYS_CLK_TCK; - erts_smp_mtx_lock(&erts_timeofday_mtx); - - prev_total_user = (t_start.tms_utime * 1000) / SYS_CLK_TCK; - prev_total_sys = (t_start.tms_stime * 1000) / SYS_CLK_TCK; - t_start = now; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); +#endif + + if (ms_user) + *ms_user = user; + if (ms_sys) + *ms_sys = sys; - if (ms_user_diff != NULL) - *ms_user_diff = total_user - prev_total_user; - - if (ms_sys_diff != NULL) - *ms_sys_diff = total_sys - prev_total_sys; + if (ms_user_diff || ms_sys_diff) { + + erts_mtx_lock(&runtime_prev.data.mtx); + + prev_user = runtime_prev.data.user; + prev_sys = runtime_prev.data.sys; + runtime_prev.data.user = user; + runtime_prev.data.sys = sys; + + erts_mtx_unlock(&runtime_prev.data.mtx); + + if (ms_user_diff) + *ms_user_diff = user - prev_user; + if (ms_sys_diff) + *ms_sys_diff = sys - prev_sys; + } } /* wall clock routines */ void -wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff) +erts_wall_clock_elapsed_both(ErtsMonotonicTime *ms_total, ErtsMonotonicTime *ms_diff) { ErtsMonotonicTime now, elapsed; - erts_smp_mtx_lock(&erts_timeofday_mtx); - now = time_sup.r.o.get_time(); update_last_mtime(NULL, now); elapsed = ERTS_MONOTONIC_TO_MSEC(now); - *ms_total = (UWord) elapsed; - *ms_diff = (UWord) (elapsed - prev_wall_clock_elapsed); - prev_wall_clock_elapsed = elapsed; + elapsed -= ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN); + + *ms_total = elapsed; - erts_smp_mtx_unlock(&erts_timeofday_mtx); + if (ms_diff) { + ErtsMonotonicTime prev; + + prev = ((ErtsMonotonicTime) + erts_atomic64_xchg_mb(&wall_clock_prev.time, + (erts_aint64_t) elapsed)); + + *ms_diff = elapsed - prev; + } } /* get current time */ @@ -1713,22 +1776,27 @@ univ_to_local(Sint *year, Sint *month, Sint *day, void get_now(Uint* megasec, Uint* sec, Uint* microsec) { - ErtsMonotonicTime now_megasec, now_sec, now, mtime, time_offset; + ErtsMonotonicTime now_megasec, now_sec, now, prev, mtime, time_offset; mtime = time_sup.r.o.get_time(); time_offset = get_time_offset(); update_last_mtime(NULL, mtime); now = ERTS_MONOTONIC_TO_USEC(mtime + time_offset); - erts_smp_mtx_lock(&erts_timeofday_mtx); - /* Make sure now time is later than last time */ - if (now <= previous_now) - now = previous_now + 1; - - previous_now = now; - - erts_smp_mtx_unlock(&erts_timeofday_mtx); + prev = erts_atomic64_read_nob(&now_prev.time); + while (1) { + ErtsMonotonicTime act; + if (now <= prev) + now = prev + 1; + act = ((ErtsMonotonicTime) + erts_atomic64_cmpxchg_mb(&now_prev.time, + (erts_aint64_t) now, + (erts_aint64_t) prev)); + if (act == prev) + break; + prev = act; + } now_megasec = now / ERTS_MONOTONIC_TIME_TERA; now_sec = now / ERTS_MONOTONIC_TIME_MEGA; @@ -1815,10 +1883,10 @@ void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec) { void erts_monitor_time_offset(Eterm id, Eterm ref) { - erts_smp_mtx_lock(&erts_get_time_mtx); + erts_mtx_lock(&erts_get_time_mtx); erts_add_monitor(&time_offset_monitors, MON_TIME_OFFSET, ref, id, NIL); no_time_offset_monitors++; - erts_smp_mtx_unlock(&erts_get_time_mtx); + erts_mtx_unlock(&erts_get_time_mtx); } int @@ -1827,7 +1895,7 @@ erts_demonitor_time_offset(Eterm ref) int res; ErtsMonitor *mon; ASSERT(is_internal_ref(ref)); - erts_smp_mtx_lock(&erts_get_time_mtx); + erts_mtx_lock(&erts_get_time_mtx); if (is_internal_ordinary_ref(ref)) mon = erts_remove_monitor(&time_offset_monitors, ref); else @@ -1839,7 +1907,7 @@ erts_demonitor_time_offset(Eterm ref) no_time_offset_monitors--; res = 1; } - erts_smp_mtx_unlock(&erts_get_time_mtx); + erts_mtx_unlock(&erts_get_time_mtx); if (res) erts_destroy_monitor(mon); return res; @@ -1897,7 +1965,7 @@ send_time_offset_changed_notifications(void *new_offsetp) #endif new_offset -= ERTS_MONOTONIC_OFFSET_NATIVE; - erts_smp_mtx_lock(&erts_get_time_mtx); + erts_mtx_lock(&erts_get_time_mtx); no_monitors = no_time_offset_monitors; if (no_monitors) { @@ -1922,7 +1990,7 @@ send_time_offset_changed_notifications(void *new_offsetp) ASSERT(cntxt.ix == no_monitors); } - erts_smp_mtx_unlock(&erts_get_time_mtx); + erts_mtx_unlock(&erts_get_time_mtx); if (no_monitors) { Eterm *hp, *patch_refp, new_offset_term, message_template; @@ -1955,7 +2023,7 @@ send_time_offset_changed_notifications(void *new_offsetp) if (rp) { Eterm ref = to_mon_info[mix].ref; ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK; - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (erts_lookup_monitor(ERTS_P_MONITORS(rp), ref)) { ErtsMessage *mp; ErlOffHeap *ohp; @@ -1968,7 +2036,7 @@ send_time_offset_changed_notifications(void *new_offsetp) message = copy_struct(message_template, hsz, &hp, ohp); erts_queue_message(rp, rp_locks, mp, message, am_clock_service); } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4b06c55770..4b996d8fc2 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -77,8 +77,8 @@ static Eterm system_profile; int erts_cpu_timestamp; #endif -static erts_smp_mtx_t smq_mtx; -static erts_smp_rwmtx_t sys_trace_rwmtx; +static erts_mtx_t smq_mtx; +static erts_rwmtx_t sys_trace_rwmtx; enum ErtsSysMsgType { SYS_MSG_TYPE_UNDEFINED, @@ -237,7 +237,6 @@ write_timestamp(ErtsTraceTimeStamp *tsp, Eterm **hpp) } } -#ifdef ERTS_SMP static ERTS_INLINE Uint patch_ts_size(int ts_type) @@ -257,7 +256,6 @@ patch_ts_size(int ts_type) return 0; } } -#endif /* ERTS_SMP */ /* * Write a timestamp. The timestamp MUST be the last @@ -298,18 +296,11 @@ write_ts(int ts_type, Eterm *hp, ErlHeapFragment *bp, Process *tracer) if (shrink) { if (bp) bp->used_size -= shrink; -#ifndef ERTS_SMP - else if (tracer) { - Eterm *endp = ts_hp + shrink; - HRelease(tracer, endp, ts_hp); - } -#endif } return res; } -#ifdef ERTS_SMP static void enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, Eterm from, Eterm to, @@ -321,7 +312,6 @@ static void enqueue_sys_msg(enum ErtsSysMsgType type, Eterm msg, ErlHeapFragment *bp); static void init_sys_msg_dispatcher(void); -#endif static void init_tracer_nif(void); static int tracer_cmp_fun(void*, void*); @@ -332,11 +322,12 @@ static void tracer_free_fun(void*); typedef struct ErtsTracerNif_ ErtsTracerNif; void erts_init_trace(void) { - erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_rwmtx_opt_t rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers"); + erts_rwmtx_init_opt(&sys_trace_rwmtx, &rwmtx_opts, "sys_tracers", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); #ifdef HAVE_ERTS_NOW_CPU erts_cpu_timestamp = 0; @@ -349,9 +340,7 @@ void erts_init_trace(void) { default_port_trace_flags = F_INITIAL_TRACE_FLAGS; default_port_tracer = erts_tracer_nil; system_seq_tracer = erts_tracer_nil; -#ifdef ERTS_SMP init_sys_msg_dispatcher(); -#endif init_tracer_nif(); } @@ -411,43 +400,35 @@ static Uint active_sched; void erts_system_profile_setup_active_schedulers(void) { - ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); active_sched = erts_active_schedulers(); } static void exiting_reset(Eterm exiting) { - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); if (exiting == system_monitor) { -#ifdef ERTS_SMP system_monitor = NIL; /* Let the trace message dispatcher clear flags, etc */ -#else - erts_system_monitor_clear(NULL); -#endif } if (exiting == system_profile) { -#ifdef ERTS_SMP system_profile = NIL; /* Let the trace message dispatcher clear flags, etc */ -#else - erts_system_profile_clear(NULL); -#endif } - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); } void erts_trace_check_exiting(Eterm exiting) { int reset = 0; - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); if (exiting == system_monitor) reset = 1; else if (exiting == system_profile) reset = 1; - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); if (reset) exiting_reset(exiting); } @@ -467,7 +448,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new } } - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); old = system_seq_tracer; system_seq_tracer = erts_tracer_nil; erts_tracer_update(&system_seq_tracer, new); @@ -475,7 +456,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "set seq tracer new=%T old=%T\n", new, old); #endif - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); return old; } @@ -483,12 +464,12 @@ ErtsTracer erts_get_system_seq_tracer(void) { ErtsTracer st; - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); st = system_seq_tracer; #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "get seq tracer %T\n", st); #endif - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); if (st != erts_tracer_nil && call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED, @@ -521,8 +502,8 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp, ErtsTracer curr_default_tracer = *default_tracer; if (tracerp) { /* we only have a rlock, so we have to unlock and then rwlock */ - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); } /* check if someone else changed default tracer while we got the write lock, if so we don't do @@ -532,8 +513,8 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp, ERTS_TRACER_CLEAR(default_tracer); } if (tracerp) { - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); } } } @@ -566,81 +547,81 @@ void erts_change_default_proc_tracing(int setflags, Uint flagsp, const ErtsTracer tracer) { - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); erts_change_default_tracing( setflags, flagsp, tracer, &default_proc_trace_flags, &default_proc_tracer); - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); } void erts_change_default_port_tracing(int setflags, Uint flagsp, const ErtsTracer tracer) { - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); erts_change_default_tracing( setflags, flagsp, tracer, &default_port_trace_flags, &default_port_tracer); - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); } void erts_get_default_proc_tracing(Uint *flagsp, ErtsTracer *tracerp) { - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); *tracerp = erts_tracer_nil; /* initialize */ get_default_tracing( flagsp, tracerp, &default_proc_trace_flags, &default_proc_tracer); - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); } void erts_get_default_port_tracing(Uint *flagsp, ErtsTracer *tracerp) { - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); *tracerp = erts_tracer_nil; /* initialize */ get_default_tracing( flagsp, tracerp, &default_port_trace_flags, &default_port_tracer); - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); } void erts_set_system_monitor(Eterm monitor) { - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); system_monitor = monitor; - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_monitor(void) { Eterm monitor; - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); monitor = system_monitor; - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); return monitor; } /* Performance monitoring */ void erts_set_system_profile(Eterm profile) { - erts_smp_rwmtx_rwlock(&sys_trace_rwmtx); + erts_rwmtx_rwlock(&sys_trace_rwmtx); system_profile = profile; - erts_smp_rwmtx_rwunlock(&sys_trace_rwmtx); + erts_rwmtx_rwunlock(&sys_trace_rwmtx); } Eterm erts_get_system_profile(void) { Eterm profile; - erts_smp_rwmtx_rlock(&sys_trace_rwmtx); + erts_rwmtx_rlock(&sys_trace_rwmtx); profile = system_profile; - erts_smp_rwmtx_runlock(&sys_trace_rwmtx); + erts_rwmtx_runlock(&sys_trace_rwmtx); return profile; } @@ -678,71 +659,11 @@ write_sys_msg_to_port(Eterm unused_to, erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer); } -#ifndef ERTS_SMP - if (!INVALID_TRACER_PORT(trace_port, trace_port->common.id)) -#endif erts_raw_port_command(trace_port, buffer, ptr-buffer); erts_free(ERTS_ALC_T_TMP, (void *) buffer); } -#ifndef ERTS_SMP -/* Profile send - * Checks if profiler is port or process - * Eterm msg is local, need copying. - */ - -static void -profile_send(Eterm from, Eterm message) { - Uint sz = 0; - Uint *hp = NULL; - Eterm msg = NIL; - Process *profile_p = NULL; - - Eterm profiler = erts_get_system_profile(); - - /* do not profile profiler pid */ - if (from == profiler) return; - - if (is_internal_port(profiler)) { - Port *profiler_port = NULL; - - /* not smp */ - - profiler_port = erts_id2port_sflgs(profiler, - NULL, - 0, - ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP); - if (profiler_port) { - write_sys_msg_to_port(profiler, - profiler_port, - NIL, /* or current process->common.id */ - SYS_MSG_TYPE_SYSPROF, - message); - erts_port_release(profiler_port); - } - - } else { - ErtsMessage *mp; - ASSERT(is_internal_pid(profiler)); - - profile_p = erts_proc_lookup(profiler); - - if (!profile_p) - return; - - sz = size_object(message); - mp = erts_alloc_message(sz, &hp); - if (sz == 0) - msg = message; - else - msg = copy_struct(message, sz, &hp, &mp->hfrag.off_heap); - - erts_queue_message(profile_p, 0, mp, msg, from); - } -} - -#endif static void trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) @@ -814,9 +735,7 @@ trace_send(Process *p, Eterm to, Eterm msg) ErtsTracerNif *tnif = NULL; ErtsTracingEvent* te; Eterm pam_result; -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl; -#endif ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); @@ -841,9 +760,7 @@ trace_send(Process *p, Eterm to, Eterm msg) } else pam_result = am_true; -#ifdef ERTS_SMP dhndl = erts_thr_progress_unmanaged_delay(); -#endif if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) @@ -861,9 +778,7 @@ trace_send(Process *p, Eterm to, Eterm msg) operation, msg, to, pam_result); } -#ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); -#endif erts_match_set_release_result_trace(p, pam_result); } @@ -1178,7 +1093,7 @@ erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec, Eterm transformed_args[MAX_ARG]; ErtsTracer pre_ms_tracer = erts_tracer_nil; - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN); ASSERT(tracer); if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { @@ -1466,21 +1381,11 @@ monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, { ErlHeapFragment *bp; ErlOffHeap *off_heap; -#ifndef ERTS_SMP - Process *monitor_p; -#endif Uint hsz; Eterm *hp, list, in_mfa = am_undefined, out_mfa = am_undefined; Eterm in_tpl, out_tpl, tmo_tpl, tmo, msg; -#ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor)); - monitor_p = erts_proc_lookup(system_monitor); - if (!monitor_p || p == monitor_p) { - return; - } -#endif /* * Size: {monitor, pid, long_schedule, [{timeout, T}, {in, {M,F,A}},{out,{M,F,A}}]} -> * 5 (top tuple of 4), (3 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + size of Timeout + @@ -1516,36 +1421,18 @@ monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, hp += 2; msg = TUPLE4(hp, am_monitor, p->common.id, am_long_schedule, list); hp += 5; -#ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(monitor_p, 0, mp, msg, am_system); - } -#endif } void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; -#ifndef ERTS_SMP - Process *monitor_p; -#endif Uint hsz; Eterm *hp, list, op; Eterm op_tpl, tmo_tpl, tmo, msg; -#ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor)); - monitor_p = erts_proc_lookup(system_monitor); - if (!monitor_p) { - return; - } -#endif /* * Size: {monitor, port, long_schedule, [{timeout, T}, {op, Operation}]} -> * 5 (top tuple of 4), (2 (elements) * 2 (cons)) + 3 (timeout tuple of 2) @@ -1562,7 +1449,6 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break; case ERTS_PORT_TASK_INPUT: op = am_input; break; case ERTS_PORT_TASK_OUTPUT: op = am_output; break; - case ERTS_PORT_TASK_EVENT: op = am_event; break; case ERTS_PORT_TASK_DIST_CMD: op = am_dist_cmd; break; default: op = am_undefined; break; } @@ -1581,24 +1467,13 @@ monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) hp += 2; msg = TUPLE4(hp, am_monitor, pp->common.id, am_long_schedule, list); hp += 5; -#ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(monitor_p, 0, mp, msg, am_system); - } -#endif } void monitor_long_gc(Process *p, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; -#ifndef ERTS_SMP - Process *monitor_p; -#endif Uint hsz; Eterm *hp, list, msg; Eterm tags[] = { @@ -1623,12 +1498,6 @@ monitor_long_gc(Process *p, Uint time) { Eterm *hp_end; #endif -#ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor)); - monitor_p = erts_proc_lookup(system_monitor); - if (!monitor_p || p == monitor_p) - return; -#endif hsz = 0; (void) erts_bld_atom_uword_2tup_list(NULL, @@ -1656,24 +1525,13 @@ monitor_long_gc(Process *p, Uint time) { ASSERT(hp == hp_end); #endif -#ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(monitor_p, 0, mp, msg, am_system); - } -#endif } void monitor_large_heap(Process *p) { ErlHeapFragment *bp; ErlOffHeap *off_heap; -#ifndef ERTS_SMP - Process *monitor_p; -#endif Uint hsz; Eterm *hp, list, msg; Eterm tags[] = { @@ -1697,13 +1555,6 @@ monitor_large_heap(Process *p) { #endif -#ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor)); - monitor_p = erts_proc_lookup(system_monitor); - if (!monitor_p || p == monitor_p) { - return; - } -#endif hsz = 0; (void) erts_bld_atom_uword_2tup_list(NULL, @@ -1731,47 +1582,22 @@ monitor_large_heap(Process *p) { ASSERT(hp == hp_end); #endif -#ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(monitor_p, 0, mp, msg, am_system); - } -#endif } void monitor_generic(Process *p, Eterm type, Eterm spec) { ErlHeapFragment *bp; ErlOffHeap *off_heap; -#ifndef ERTS_SMP - Process *monitor_p; -#endif Eterm *hp, msg; -#ifndef ERTS_SMP - ASSERT(is_internal_pid(system_monitor)); - monitor_p = erts_proc_lookup(system_monitor); - if (!monitor_p || p == monitor_p) - return; -#endif hp = ERTS_ALLOC_SYSMSG_HEAP(5, &bp, &off_heap, monitor_p); msg = TUPLE4(hp, am_monitor, p->common.id, type, spec); hp += 5; -#ifdef ERTS_SMP enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(monitor_p, 0, mp, msg, am_system); - } -#endif } @@ -1784,21 +1610,14 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { Eterm *hp, msg; ErlHeapFragment *bp = NULL; -#ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (7 + ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - hp = local_heap; -#else Uint hsz; hsz = 7 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; -#endif - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); switch (state) { case am_active: @@ -1820,14 +1639,8 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { /* Write timestamp in element 6 of the 'msg' tuple */ hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); -#ifndef ERTS_SMP - profile_send(NIL, msg); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp); -#endif - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } @@ -1836,7 +1649,7 @@ profile_scheduler(Eterm scheduler_id, Eterm state) { void trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) { ErtsTracerNif *tnif = NULL; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &p->common, &tnif, TRACE_FUN_E_PORTS, am_open)) send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PORTS, am_open, calling_pid, drv_name, am_true); @@ -1853,9 +1666,9 @@ void trace_port(Port *t_p, Eterm what, Eterm data) { ErtsTracerNif *tnif = NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PORTS, what)) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PORTS, what, data, THE_NON_VALUE, am_true); @@ -1898,9 +1711,9 @@ void trace_port_receive(Port *t_p, Eterm caller, Eterm what, ...) { ErtsTracerNif *tnif = NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_RECEIVE, am_receive)) { /* We can use a stack heap here, as the nif is called in the context of a port */ @@ -2015,9 +1828,9 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists) { ErtsTracerNif *tnif = NULL; Eterm op = exists ? am_send : am_send_to_non_existing_process; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, op)) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SEND, op, msg, receiver, am_true); @@ -2026,9 +1839,9 @@ trace_port_send(Port *t_p, Eterm receiver, Eterm msg, int exists) void trace_port_send_binary(Port *t_p, Eterm to, Eterm what, char *bin, Sint sz) { ErtsTracerNif *tnif = NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SEND, am_send)) { Eterm msg; Binary* bptr = NULL; @@ -2074,9 +1887,9 @@ trace_sched_ports(Port *p, Eterm what) { void trace_sched_ports_where(Port *t_p, Eterm what, Eterm where) { ErtsTracerNif *tnif = NULL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_thr_progress_is_blocking()); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_SCHED_PORT, what)) send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_SCHED_PORT, @@ -2091,24 +1904,14 @@ profile_runnable_port(Port *p, Eterm status) { ErlHeapFragment *bp = NULL; Eterm count = make_small(0); -#ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) - - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; - -#else Uint hsz; hsz = 6 + patch_ts_size(erts_system_profile_ts_type)-1; bp = new_message_buffer(hsz); hp = bp->mem; -#endif - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); msg = TUPLE5(hp, am_profile, p->common.id, status, count, NIL /* Will be overwritten by timestamp */); @@ -2117,14 +1920,8 @@ profile_runnable_port(Port *p, Eterm status) { /* Write timestamp in element 5 of the 'msg' tuple */ hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); -#ifndef ERTS_SMP - profile_send(p->common.id, msg); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp); -#endif - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } /* Process profiling */ @@ -2135,23 +1932,13 @@ profile_runnable_proc(Process *p, Eterm status){ ErlHeapFragment *bp = NULL; ErtsCodeMFA *cmfa = NULL; -#ifndef ERTS_SMP -#define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) - DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); - UseTmpHeapNoproc(LOCAL_HEAP_SIZE); - - hp = local_heap; -#else ErtsThrPrgrDelayHandle dhndl; Uint hsz = 4 + 6 + patch_ts_size(erts_system_profile_ts_type)-1; -#endif /* Assumptions: * We possibly don't have the MAIN_LOCK for the process p here. * We assume that we can read from p->current and p->i atomically */ -#ifdef ERTS_SMP dhndl = erts_thr_progress_unmanaged_delay(); /* suspend purge operations */ -#endif if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { @@ -2161,14 +1948,12 @@ profile_runnable_proc(Process *p, Eterm status){ } } -#ifdef ERTS_SMP if (!cmfa) { hsz -= 4; } bp = new_message_buffer(hsz); hp = bp->mem; -#endif if (cmfa) { where = TUPLE3(hp, cmfa->module, cmfa->function, @@ -2178,11 +1963,9 @@ profile_runnable_proc(Process *p, Eterm status){ where = make_small(0); } -#ifdef ERTS_SMP erts_thr_progress_unmanaged_continue(dhndl); -#endif - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); msg = TUPLE5(hp, am_profile, p->common.id, status, where, NIL /* Will be overwritten by timestamp */); @@ -2191,20 +1974,13 @@ profile_runnable_proc(Process *p, Eterm status){ /* Write timestamp in element 5 of the 'msg' tuple */ hp[-1] = write_ts(erts_system_profile_ts_type, hp, bp, NULL); -#ifndef ERTS_SMP - profile_send(p->common.id, msg); - UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE); -#undef LOCAL_HEAP_SIZE -#else enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->common.id, NIL, msg, bp); -#endif - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } /* End system_profile tracing */ -#ifdef ERTS_SMP typedef struct ErtsSysMsgQ_ ErtsSysMsgQ; struct ErtsSysMsgQ_ { @@ -2250,7 +2026,7 @@ enqueue_sys_msg_unlocked(enum ErtsSysMsgType type, sys_message_queue = smqp; } sys_message_queue_end = smqp; - erts_smp_cnd_signal(&smq_cnd); + erts_cnd_signal(&smq_cnd); } static void @@ -2260,9 +2036,9 @@ enqueue_sys_msg(enum ErtsSysMsgType type, Eterm msg, ErlHeapFragment *bp) { - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); enqueue_sys_msg_unlocked(type, from, to, msg, bp); - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } void @@ -2314,10 +2090,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_monitor_flags.busy_port && !erts_system_monitor_flags.busy_dist_port) break; /* Everything is disabled */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); if (system_monitor == receiver || receiver == NIL) erts_system_monitor_clear(NULL); - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); break; case SYS_MSG_TYPE_SYSPROF: if (receiver == NIL @@ -2327,11 +2103,11 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) && !erts_system_profile_flags.scheduler) break; /* Block system to clear flags */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); if (system_profile == receiver || receiver == NIL) { erts_system_profile_clear(NULL); } - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); break; case SYS_MSG_TYPE_ERRLGR: { char *no_elgger = "(no error logger present)"; @@ -2376,38 +2152,38 @@ static void sys_msg_dispatcher_wakeup(void *vwait_p) { int *wait_p = (int *) vwait_p; - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); *wait_p = 0; - erts_smp_cnd_signal(&smq_cnd); - erts_smp_mtx_unlock(&smq_mtx); + erts_cnd_signal(&smq_cnd); + erts_mtx_unlock(&smq_mtx); } static void sys_msg_dispatcher_prep_wait(void *vwait_p) { int *wait_p = (int *) vwait_p; - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); *wait_p = 1; - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } static void sys_msg_dispatcher_fin_wait(void *vwait_p) { int *wait_p = (int *) vwait_p; - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); *wait_p = 0; - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } static void sys_msg_dispatcher_wait(void *vwait_p) { int *wait_p = (int *) vwait_p; - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); while (*wait_p) - erts_smp_cnd_wait(&smq_cnd, &smq_mtx); - erts_smp_mtx_unlock(&smq_mtx); + erts_cnd_wait(&smq_cnd, &smq_mtx); + erts_mtx_unlock(&smq_mtx); } static void * @@ -2433,9 +2209,9 @@ sys_msg_dispatcher_func(void *unused) int end_wait = 0; ErtsSysMsgQ *smqp; - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); /* Free previously used queue ... */ while (local_sys_message_queue) { @@ -2446,21 +2222,21 @@ sys_msg_dispatcher_func(void *unused) /* Fetch current trace message queue ... */ if (!sys_message_queue) { - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); end_wait = 1; erts_thr_progress_active(NULL, 0); erts_thr_progress_prepare_wait(NULL); - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); } while (!sys_message_queue) - erts_smp_cnd_wait(&smq_cnd, &smq_mtx); + erts_cnd_wait(&smq_cnd, &smq_mtx); local_sys_message_queue = sys_message_queue; sys_message_queue = NULL; sys_message_queue_end = NULL; - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); if (end_wait) { erts_thr_progress_finalize_wait(NULL); @@ -2534,7 +2310,7 @@ sys_msg_dispatcher_func(void *unused) #ifdef DEBUG_PRINTOUTS erts_fprintf(stderr, "delivered\n"); #endif - erts_smp_proc_unlock(proc, proc_locks); + erts_proc_unlock(proc, proc_locks); } } else if (receiver == am_error_logger) { @@ -2572,7 +2348,7 @@ sys_msg_dispatcher_func(void *unused) sys_msg_disp_failure(smqp, receiver); drop_sys_msg: if (proc) - erts_smp_proc_unlock(proc, proc_locks); + erts_proc_unlock(proc, proc_locks); if (smqp->bp) free_message_buffer(smqp->bp); #ifdef DEBUG_PRINTOUTS @@ -2592,7 +2368,7 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm, ErlHeapFragment *)) { ErtsSysMsgQ *sm; - erts_smp_mtx_lock(&smq_mtx); + erts_mtx_lock(&smq_mtx); for (sm = sys_message_queue; sm; sm = sm->next) { Eterm to; switch (sm->type) { @@ -2611,28 +2387,28 @@ erts_foreach_sys_msg_in_q(void (*func)(Eterm, } (*func)(sm->from, to, sm->msg, sm->bp); } - erts_smp_mtx_unlock(&smq_mtx); + erts_mtx_unlock(&smq_mtx); } static void init_sys_msg_dispatcher(void) { - erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; thr_opts.detached = 1; thr_opts.name = "sys_msg_dispatcher"; init_smq_element_alloc(); sys_message_queue = NULL; sys_message_queue_end = NULL; - erts_smp_cnd_init(&smq_cnd); - erts_smp_mtx_init(&smq_mtx, "sys_msg_q"); - erts_smp_thr_create(&sys_msg_dispatcher_tid, + erts_cnd_init(&smq_cnd); + erts_mtx_init(&smq_mtx, "sys_msg_q", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); + erts_thr_create(&sys_msg_dispatcher_tid, sys_msg_dispatcher_func, NULL, &thr_opts); } -#endif #include "erl_nif.h" @@ -2728,7 +2504,7 @@ static void init_tracer_template(ErtsTracerNif *tnif) { } static Hash *tracer_hash = NULL; -static erts_smp_rwmtx_t tracer_mtx; +static erts_rwmtx_t tracer_mtx; static ErtsTracerNif * load_tracer_nif(const ErtsTracer tracer) @@ -2768,9 +2544,9 @@ load_tracer_nif(const ErtsTracer tracer) return NULL; } - erts_smp_rwmtx_rwlock(&tracer_mtx); + erts_rwmtx_rwlock(&tracer_mtx); tnif = hash_put(tracer_hash, &tnif_tmpl); - erts_smp_rwmtx_rwunlock(&tracer_mtx); + erts_rwmtx_rwunlock(&tracer_mtx); return tnif; } @@ -2781,14 +2557,14 @@ lookup_tracer_nif(const ErtsTracer tracer) ErtsTracerNif tnif_tmpl; ErtsTracerNif *tnif; tnif_tmpl.module = ERTS_TRACER_MODULE(tracer); - erts_smp_rwmtx_rlock(&tracer_mtx); + erts_rwmtx_rlock(&tracer_mtx); if ((tnif = hash_get(tracer_hash, &tnif_tmpl)) == NULL) { - erts_smp_rwmtx_runlock(&tracer_mtx); + erts_rwmtx_runlock(&tracer_mtx); tnif = load_tracer_nif(tracer); ASSERT(!tnif || tnif->nif_mod); return tnif; } - erts_smp_rwmtx_runlock(&tracer_mtx); + erts_rwmtx_runlock(&tracer_mtx); ASSERT(tnif->nif_mod); return tnif; } @@ -2926,17 +2702,17 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, Eterm t_p_id, ErtsTracerNif *tnif, enum ErtsTracerOpt topt, Eterm tag, Eterm msg, Eterm extra, Eterm pam_result) { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) if (c_p) { /* We have to hold the main lock of the currently executing process */ erts_proc_lc_chk_have_proc_locks(c_p, ERTS_PROC_LOCK_MAIN); } if (is_internal_pid(t_p->id)) { /* We have to have at least one lock */ - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL); } else { ASSERT(is_internal_port(t_p->id)); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)); + ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p)); } #endif @@ -2979,17 +2755,17 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, enum ErtsTracerOpt topt, Eterm tag) { Eterm nif_result; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) if (c_p) - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == c_p_locks || erts_thr_progress_is_blocking()); if (is_internal_pid(t_p->id)) { /* We have to have at least one lock */ - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks((Process*)t_p) & ERTS_PROC_LOCKS_ALL || erts_thr_progress_is_blocking()); } else { ASSERT(is_internal_port(t_p->id)); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p) + ERTS_LC_ASSERT(erts_lc_is_port_locked((Port*)t_p) || erts_thr_progress_is_blocking()); } #endif @@ -3011,12 +2787,12 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, if (is_internal_port(t_p->id) || (c_p && c_p->common.id == t_p->id)) { ErtsProcLocks c_p_xlocks = 0; if (is_internal_pid(t_p->id)) { - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); + ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) & ERTS_PROC_LOCK_MAIN); if (c_p_locks != ERTS_PROC_LOCKS_ALL) { c_p_xlocks = ~c_p_locks & ERTS_PROC_LOCKS_ALL; - if (erts_smp_proc_trylock(c_p, c_p_xlocks) == EBUSY) { - erts_smp_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); + if (erts_proc_trylock(c_p, c_p_xlocks) == EBUSY) { + erts_proc_unlock(c_p, c_p_locks & ~ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); } } } @@ -3024,7 +2800,7 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, t_p->trace_flags &= ~TRACEE_FLAGS; if (c_p_xlocks) - erts_smp_proc_unlock(c_p, c_p_xlocks); + erts_proc_unlock(c_p, c_p_xlocks); } return 0; @@ -3064,7 +2840,7 @@ int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks, void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer) { -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) if (is_internal_pid(t_p->id) && !erts_thr_progress_is_blocking()) { erts_proc_lc_chk_have_proc_locks((Process*)t_p, ERTS_PROC_LOCKS_ALL); } else if (is_internal_port(t_p->id)) { @@ -3182,10 +2958,12 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) static void init_tracer_nif() { - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx"); + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; + + erts_rwmtx_init_opt(&tracer_mtx, &rwmtx_opt, "tracer_mtx", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); erts_tracer_nif_clear(); @@ -3194,7 +2972,7 @@ static void init_tracer_nif() int erts_tracer_nif_clear() { - erts_smp_rwmtx_rlock(&tracer_mtx); + erts_rwmtx_rlock(&tracer_mtx); if (!tracer_hash || tracer_hash->nobjs) { HashFunctions hf; @@ -3206,19 +2984,19 @@ int erts_tracer_nif_clear() hf.meta_free = (HMFREE_FUN) erts_free; hf.meta_print = (HMPRINT_FUN) erts_print; - erts_smp_rwmtx_runlock(&tracer_mtx); - erts_smp_rwmtx_rwlock(&tracer_mtx); + erts_rwmtx_runlock(&tracer_mtx); + erts_rwmtx_rwlock(&tracer_mtx); if (tracer_hash) hash_delete(tracer_hash); tracer_hash = hash_new(ERTS_ALC_T_TRACER_NIF, "tracer_hash", 10, hf); - erts_smp_rwmtx_rwunlock(&tracer_mtx); + erts_rwmtx_rwunlock(&tracer_mtx); return 1; } - erts_smp_rwmtx_runlock(&tracer_mtx); + erts_rwmtx_runlock(&tracer_mtx); return 0; } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 01fe1e5e23..dbf7ebd2a1 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -87,7 +87,6 @@ void erts_set_system_monitor(Eterm monitor); Eterm erts_get_system_monitor(void); int erts_is_tracer_valid(Process* p); -#ifdef ERTS_SMP void erts_check_my_tracer_proc(Process *); void erts_block_sys_msg_dispatcher(void); void erts_release_sys_msg_dispatcher(void); @@ -97,7 +96,6 @@ void erts_foreach_sys_msg_in_q(void (*func)(Eterm, ErlHeapFragment *)); void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); -#endif void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); @@ -149,16 +147,12 @@ erts_bif_trace_epilogue(Process *p, Eterm result, int applying, Uint32 flags_meta, BeamInstr* I, ErtsTracer meta_tracer); -#ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); -#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) \ +#define ERTS_CHK_PEND_TRACE_MSGS(ESDP) \ do { \ if ((ESDP)->pending_trace_msgs) \ erts_send_pending_trace_msgs((ESDP)); \ } while (0) -#else -#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP) -#endif #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 2d1d1443a7..efd2ca3db2 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -1988,7 +1988,7 @@ char *erts_convert_filename_to_encoding(Eterm name, char *statbuf, size_t statbu is_list(name) || (allow_empty && is_nil(name))) { Sint need; - if ((need = erts_native_filename_need(name,encoding)) < 0) { + if ((need = erts_native_filename_need(name, encoding, 0)) < 0) { return NULL; } if (encoding == ERL_FILENAME_WIN_WCHAR) { @@ -2152,12 +2152,13 @@ Eterm erts_convert_native_to_filename(Process *p, byte *bytes) } -Sint erts_native_filename_need(Eterm ioterm, int encoding) +Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null) { Eterm *objp; Eterm obj; DECLARE_ESTACK(stack); Sint need = 0; + int seen_null = 0; if (is_atom(ioterm)) { Atom* ap; @@ -2194,6 +2195,24 @@ Sint erts_native_filename_need(Eterm ioterm, int encoding) default: need = -1; } + if (!allow_null) { + /* + * Do not allow null in + * the middle of filenames + */ + if (need > 0) { + byte *name = ap->name; + int len = ap->len; + for (i = 0; i < len; i++) { + if (name[i] == 0) + seen_null = 1; + else if (seen_null) { + need = -1; + break; + } + } + } + } DESTROY_ESTACK(stack); return need; } @@ -2224,6 +2243,18 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ if (is_small(obj)) { /* Always small */ for(;;) { Uint x = unsigned_val(obj); + if (!allow_null) { + /* + * Do not allow null in + * the middle of filenames + */ + if (x == 0) + seen_null = 1; + else if (seen_null) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + } switch (encoding) { case ERL_FILENAME_LATIN1: if (x > 255) { @@ -2515,6 +2546,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } if (is_binary(BIF_ARG_1)) { + int seen_null = 0; byte *temp_alloc = NULL; byte *bytes; byte *err_pos; @@ -2524,10 +2556,18 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) size = binary_size(BIF_ARG_1); bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); if (encoding != ERL_FILENAME_WIN_WCHAR) { + Uint i; /*Add 0 termination only*/ bin_term = new_binary(BIF_P, NULL, size+1); bin_p = binary_bytes(bin_term); - memcpy(bin_p,bytes,size); + for (i = 0; i < size; i++) { + /* Don't allow null in the middle of filenames... */ + if (bytes[i] == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; + bin_p[i] = bytes[i]; + } bin_p[size]=0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); @@ -2541,6 +2581,11 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_term = new_binary(BIF_P, 0, (size+1)*2); bin_p = binary_bytes(bin_term); while (size--) { + /* Don't allow null in the middle of filenames... */ + if (*bytes == 0) + seen_null = 1; + else if (seen_null) + goto bin_name_error; *bin_p++ = *bytes++; *bin_p++ = 0; } @@ -2558,11 +2603,14 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) bin_p[num_chars*2+1] = 0; erts_free_aligned_binary_bytes(temp_alloc); BIF_RET(bin_term); + bin_name_error: + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); } /* binary */ - if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { - BIF_ERROR(BIF_P,BADARG); + if ((need = erts_native_filename_need(BIF_ARG_1, encoding, 0)) < 0) { + BIF_ERROR(BIF_P,BADARG); } if (encoding == ERL_FILENAME_WIN_WCHAR) { need += 2; diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 07cf4f6903..44d8c85867 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -22,15 +22,11 @@ #define ERL_UTILS_H__ #include "sys.h" -#include "erl_smp.h" #include "erl_printf.h" struct process; typedef struct { -#ifdef DEBUG - int smp_api; -#endif union { Uint64 not_atomic; erts_atomic64_t atomic; @@ -38,70 +34,25 @@ typedef struct { } erts_interval_t; void erts_interval_init(erts_interval_t *); -void erts_smp_interval_init(erts_interval_t *); Uint64 erts_step_interval_nob(erts_interval_t *); Uint64 erts_step_interval_relb(erts_interval_t *); -Uint64 erts_smp_step_interval_nob(erts_interval_t *); -Uint64 erts_smp_step_interval_relb(erts_interval_t *); Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); -Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); -Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); -ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); -ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *); -ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *); -ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE Uint64 -erts_current_interval_nob__(erts_interval_t *icp) -{ - return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); -} - -ERTS_GLB_INLINE Uint64 -erts_current_interval_acqb__(erts_interval_t *icp) -{ - return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); -} - -ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *icp) { - ASSERT(!icp->smp_api); - return erts_current_interval_nob__(icp); + return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *icp) { - ASSERT(!icp->smp_api); - return erts_current_interval_acqb__(icp); -} - -ERTS_GLB_INLINE Uint64 -erts_smp_current_interval_nob(erts_interval_t *icp) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return erts_current_interval_nob__(icp); -#else - return icp->counter.not_atomic; -#endif -} - -ERTS_GLB_INLINE Uint64 -erts_smp_current_interval_acqb(erts_interval_t *icp) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return erts_current_interval_acqb__(icp); -#else - return icp->counter.not_atomic; -#endif + return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); } #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ @@ -131,6 +82,7 @@ Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw); Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64); Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64); +#define erts_bld_monotonic_time erts_bld_sint64 Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr); Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...); #define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2) diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 0b8d78c469..661538eadd 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -46,8 +46,6 @@ */ #define ERTS_X_REGS_ALLOCATED (MAX_REG+3) -#define INPUT_REDUCTIONS (2 * CONTEXT_REDS) - #define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ #define VH_DEFAULT_SIZE 32768 /* default virtual (bin) heap min size (words) */ #define H_DEFAULT_MAX_SIZE 0 /* default max heap size is off */ @@ -55,7 +53,7 @@ #define CP_SIZE 1 #define ErtsHAllocLockCheck(P) \ - ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P))) + ERTS_LC_ASSERT(erts_dbg_check_halloc_lock((P))) #ifdef DEBUG @@ -102,9 +100,11 @@ if ((ptr) == (endp)) { \ ; \ } else if (HEAP_START(p) <= (ptr) && (ptr) < HEAP_TOP(p)) { \ + ASSERT(HEAP_TOP(p) == (endp)); \ HEAP_TOP(p) = (ptr); \ } else { \ - erts_heap_frag_shrink(p, ptr); \ + ASSERT(MBUF(p)->mem + MBUF(p)->used_size == (endp)); \ + erts_heap_frag_shrink(p, ptr); \ } #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p)) @@ -157,6 +157,7 @@ typedef struct op_entry { Uint32 mask[3]; /* Signature mask. */ unsigned involves_r; /* Needs special attention when matching. */ int sz; /* Number of loaded words. */ + int adjust; /* Adjustment for start of instruction. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ } OpEntry; @@ -199,11 +200,15 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) +#if defined(NO_JUMP_TABLE) +# define BeamOpsAreInitialized() (1) +# define BeamOpCodeAddr(OpCode) ((BeamInstr)(OpCode)) #else extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] +# define BeamOpsAreInitialized() (beam_ops != 0) +# define BeamOpCodeAddr(OpCode) ((BeamInstr)beam_ops[(OpCode)]) #endif +#define BeamIsOpCode(InstrWord, OpCode) ((InstrWord) == BeamOpCodeAddr(OpCode)) + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/erlang_lttng.h b/erts/emulator/beam/erlang_lttng.h index 4e869671f7..feb05f4f4c 100644 --- a/erts/emulator/beam/erlang_lttng.h +++ b/erts/emulator/beam/erlang_lttng.h @@ -159,21 +159,6 @@ TRACEPOINT_EVENT( TRACEPOINT_EVENT( org_erlang_otp, - driver_event, - TP_ARGS( - char*, pid, - char*, port, - char*, driver - ), - TP_FIELDS( - ctf_string(pid, pid) - ctf_string(port, port) - ctf_string(driver, driver) - ) -) - -TRACEPOINT_EVENT( - org_erlang_otp, driver_timeout, TP_ARGS( char*, pid, diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 57f5ba5436..946ffeffb8 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -41,16 +41,13 @@ static IndexTable export_tables[ERTS_NUM_CODE_IX]; /* Active not locked */ -static erts_smp_atomic_t total_entries_bytes; - -#include "erl_smp.h" +static erts_atomic_t total_entries_bytes; /* This lock protects the staging export table from concurrent access * AND it protects the staging table from becoming active. */ -erts_smp_mtx_t export_staging_lock; +erts_mtx_t export_staging_lock; -extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; struct export_entry @@ -85,17 +82,13 @@ static struct export_blob* entry_to_blob(struct export_entry* ee) void export_info(fmtfn_t to, void *to_arg) { -#ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; if (lock) export_staging_lock(); -#endif index_info(to, to_arg, &export_tables[erts_active_code_ix()]); hash_info(to, to_arg, &export_tables[erts_staging_code_ix()].htable); -#ifdef ERTS_SMP if (lock) export_staging_unlock(); -#endif } @@ -129,14 +122,17 @@ export_alloc(struct export_entry* tmpl_e) Export* obj; blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); - erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); + erts_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; obj->info.op = 0; obj->info.u.gen_bp = NULL; obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; - obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[0] = 0; + if (BeamOpsAreInitialized()) { + obj->beam[0] = BeamOpCodeAddr(op_call_error_handler); + } obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { @@ -173,7 +169,7 @@ export_free(struct export_entry* obj) } DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); - erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); + erts_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } void @@ -182,8 +178,9 @@ init_export_table(void) HashFunctions f; int i; - erts_smp_mtx_init(&export_staging_lock, "export_tab"); - erts_smp_atomic_init_nob(&total_entries_bytes, 0); + erts_mtx_init(&export_staging_lock, "export_tab", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); + erts_atomic_init_nob(&total_entries_bytes, 0); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; @@ -272,7 +269,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || (ee->ep->addressv[code_ix] == ee->ep->beam && - ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) { return NULL; } return ee->ep; @@ -372,7 +369,7 @@ int export_table_sz(void) } int export_entries_sz(void) { - return erts_smp_atomic_read_nob(&total_entries_bytes); + return erts_atomic_read_nob(&total_entries_bytes); } Export *export_get(Export *e) { diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 7c812b306c..194e514b12 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -66,14 +66,14 @@ Export *export_get(Export*); void export_start_staging(void); void export_end_staging(int commit); -extern erts_smp_mtx_t export_staging_lock; -#define export_staging_lock() erts_smp_mtx_lock(&export_staging_lock) -#define export_staging_unlock() erts_smp_mtx_unlock(&export_staging_lock) +extern erts_mtx_t export_staging_lock; +#define export_staging_lock() erts_mtx_lock(&export_staging_lock) +#define export_staging_unlock() erts_mtx_unlock(&export_staging_lock) #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ (((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ - ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) + (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif))) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1560844521..970158933f 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -616,7 +616,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) sys_memcpy((void *) ep, (void *) edep, dist_ext_sz); ep += dist_ext_sz; if (new_edep->dep) - erts_smp_refc_inc(&new_edep->dep->refc, 1); + erts_ref_dist_entry(new_edep->dep); new_edep->extp = ep; new_edep->ext_endp = ep + ext_sz; new_edep->heap_size = -1; @@ -629,7 +629,8 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, byte *ext, Uint size, DistEntry *dep, - ErtsAtomCache *cache) + ErtsAtomCache *cache, + Uint32 *connection_id) { #undef ERTS_EXT_FAIL #undef ERTS_EXT_HDR_FAIL @@ -650,33 +651,36 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, if (size < 2) ERTS_EXT_FAIL; + if (!dep) + ERTS_INTERNAL_ERROR("Invalid use"); + if (ep[0] != VERSION_MAGIC) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - if (dep) - erts_dsprintf(dsbufp, - "** Got message from incompatible erlang on " - "channel %d\n", - dist_entry_channel_no(dep)); - else - erts_dsprintf(dsbufp, - "** Attempt to convert old incompatible " - "binary %d\n", - *ep); + erts_dsprintf(dsbufp, + "** Got message from incompatible erlang on " + "channel %d\n", + dist_entry_channel_no(dep)); erts_send_error_to_logger_nogl(dsbufp); ERTS_EXT_FAIL; } edep->flags = 0; edep->dep = dep; - if (dep) { - erts_smp_de_rlock(dep); - if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) - edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; - - edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK); - erts_smp_de_runlock(dep); + + erts_de_rlock(dep); + + if ((dep->status & (ERTS_DE_SFLG_EXITING|ERTS_DE_SFLG_CONNECTED)) + != ERTS_DE_SFLG_CONNECTED) { + erts_de_runlock(dep); + return ERTS_PREP_DIST_EXT_CLOSED; } + if (dep->flags & DFLAG_DIST_HDR_ATOM_CACHE) + edep->flags |= ERTS_DIST_EXT_DFLAG_HDR; + + *connection_id = dep->connection_id; + edep->flags |= (dep->connection_id & ERTS_DIST_EXT_CON_ID_MASK); + if (ep[1] != DIST_HEADER) { if (edep->flags & ERTS_DIST_EXT_DFLAG_HDR) ERTS_EXT_HDR_FAIL; @@ -835,14 +839,15 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, ERTS_EXT_FAIL; #endif - return 0; + erts_de_runlock(dep); + + return ERTS_PREP_DIST_EXT_SUCCESS; #undef CHKSIZE #undef ERTS_EXT_FAIL #undef ERTS_EXT_HDR_FAIL - bad_hdr: - if (dep) { + bad_hdr: { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "%T got a corrupted distribution header from %T " @@ -855,10 +860,11 @@ erts_prepare_dist_ext(ErtsDistExternal *edep, erts_dsprintf(dsbufp, ">>"); erts_send_warning_to_logger_nogl(dsbufp); } - fail: - if (dep) - erts_kill_dist_connection(dep, dep->connection_id); - return -1; + fail: { + erts_de_runlock(dep); + erts_kill_dist_connection(dep, *connection_id); + } + return ERTS_PREP_DIST_EXT_FAILED; } static void @@ -1923,7 +1929,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla } result_bin = erts_bin_nrml_alloc(size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte)VERSION_MAGIC; /* Next state immediately, no need to export context */ context->state = TTBEncode; context->s.ec.flags = flags; @@ -1981,7 +1987,7 @@ static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint fla context->s.cc.result_bin = result_bin; result_bin = erts_bin_nrml_alloc(real_size); - result_bin->orig_bytes[0] = VERSION_MAGIC; + result_bin->orig_bytes[0] = (byte) VERSION_MAGIC; context->s.cc.destination_bin = result_bin; context->s.cc.dest_len = 0; @@ -3109,7 +3115,7 @@ dec_term(ErtsDistExternal *edep, #if defined(ARCH_64) *objp = make_small(sn); #else - if (MY_IS_SSMALL(sn)) { + if (IS_SSMALL(sn)) { *objp = make_small(sn); } else { *objp = small_to_big(sn, hp); diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index f00426cc16..3c61d013da 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -185,8 +185,13 @@ ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *); ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint); void *erts_dist_ext_trailer(ErtsDistExternal *); void erts_destroy_dist_ext_copy(ErtsDistExternal *); + +#define ERTS_PREP_DIST_EXT_FAILED (-1) +#define ERTS_PREP_DIST_EXT_SUCCESS (0) +#define ERTS_PREP_DIST_EXT_CLOSED (1) + int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, - DistEntry *, ErtsAtomCache *); + DistEntry *, ErtsAtomCache *, Uint32 *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); diff --git a/erts/emulator/beam/float_instrs.tab b/erts/emulator/beam/float_instrs.tab new file mode 100644 index 0000000000..3d4db77892 --- /dev/null +++ b/erts/emulator/beam/float_instrs.tab @@ -0,0 +1,88 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +LOAD_DOUBLE(Src, Dst) { + GET_DOUBLE($Src, *(FloatDef *) &$Dst); +} + +fload(Reg, Dst) { + $LOAD_DOUBLE($Reg, $Dst); +} + +fstore(Float, Dst) { + PUT_DOUBLE(*((FloatDef *) &$Float), HTOP); + $Dst = make_float(HTOP); + HTOP += FLOAT_SIZE_OBJECT; +} + +fconv(Src, Dst) { + Eterm src = $Src; + + if (is_small(src)) { + $Dst = (double) signed_val(src); + } else if (is_big(src)) { + if (big_to_double(src, &$Dst) < 0) { + $BADARITH0(); + } + } else if (is_float(src)) { + $LOAD_DOUBLE(src, $Dst); + } else { + $BADARITH0(); + } +} + +FLOAT_OP(Src1, OP, Src2, Dst) { + ERTS_NO_FPE_CHECK_INIT(c_p); + $Dst = $Src1 $OP $Src2; + ERTS_NO_FPE_ERROR(c_p, $Dst, $BADARITH0()); +} + +i_fadd(Src1, Src2, Dst) { + $FLOAT_OP($Src1, +, $Src2, $Dst); +} + +i_fsub(Src1, Src2, Dst) { + $FLOAT_OP($Src1, -, $Src2, $Dst); +} + +i_fmul(Src1, Src2, Dst) { + $FLOAT_OP($Src1, *, $Src2, $Dst); +} + +i_fdiv(Src1, Src2, Dst) { + $FLOAT_OP($Src1, /, $Src2, $Dst); +} + +i_fnegate(Src, Dst) { + ERTS_NO_FPE_CHECK_INIT(c_p); + $Dst = -$Src; + ERTS_NO_FPE_ERROR(c_p, $Dst, $BADARITH0()); +} + +%unless NO_FPE_SIGNALS +fclearerror() { + ERTS_FP_CHECK_INIT(c_p); +} + +i_fcheckerror() { + ERTS_FP_ERROR(c_p, freg[0].fd, $BADARITH0()); +} +%endif diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index fc95535ec3..604172857a 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -85,7 +85,7 @@ struct enif_resource_type_t typedef struct { - erts_smp_mtx_t lock; + erts_mtx_t lock; ErtsMonitor* root; int pending_failed_fire; int is_dying; @@ -128,10 +128,8 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_func_t *, int argc, Eterm *argv); -#ifdef ERTS_DIRTY_SCHEDULERS int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg); -#endif /* ERTS_DIRTY_SCHEDULERS */ /* Driver handle (wrapper for old plain handle) */ @@ -183,9 +181,9 @@ typedef struct { void *handle; /* Handle for DLL or SO (for dyn. drivers). */ DE_ProcEntry *procs; /* List of pids that have loaded this driver, or that wait for it to change state */ - erts_smp_refc_t refc; /* Number of ports/processes having + erts_refc_t refc; /* Number of ports/processes having references to the driver */ - erts_smp_atomic32_t port_count; /* Number of ports using the driver */ + erts_atomic32_t port_count; /* Number of ports using the driver */ Uint flags; /* ERL_DE_FL_KILL_PORTS */ int status; /* ERL_DE_xxx */ char *full_path; /* Full path of the driver */ @@ -209,9 +207,7 @@ struct erts_driver_t_ { } version; int flags; DE_Handle *handle; -#ifdef ERTS_SMP - erts_smp_mtx_t *lock; -#endif + erts_mtx_t *lock; ErlDrvEntry *entry; ErlDrvData (*start)(ErlDrvPort port, char *command, SysDriverOpts* opts); void (*stop)(ErlDrvData drv_data); @@ -226,8 +222,6 @@ struct erts_driver_t_ { char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen, /* Might be NULL */ unsigned int *flags); - void (*event)(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event); void (*timeout)(ErlDrvData drv_data); @@ -238,7 +232,7 @@ struct erts_driver_t_ { }; extern erts_driver_t *driver_list; -extern erts_smp_rwmtx_t erts_driver_list_lock; +extern erts_rwmtx_t erts_driver_list_lock; extern void erts_ddll_init(void); extern void erts_ddll_lock_driver(DE_Handle *dh, char *name); @@ -299,7 +293,7 @@ extern Eterm node_cookie; extern Uint display_items; /* no of items to display in traces etc */ extern int erts_backtrace_depth; -extern erts_smp_atomic32_t erts_max_gen_gcs; +extern erts_atomic32_t erts_max_gen_gcs; extern int bif_reductions; /* reductions + fcalls (when doing call_bif) */ extern int stackdump_on_exit; @@ -909,13 +903,11 @@ typedef struct ErtsLiteralArea_ { #define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) -extern erts_smp_atomic_t erts_copy_literal_area__; +extern erts_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ - ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__)) + ((ErtsLiteralArea *) erts_atomic_read_nob(&erts_copy_literal_area__)) extern Process *erts_literal_area_collector; -#ifdef ERTS_DIRTY_SCHEDULERS extern Process *erts_dirty_process_code_checker; -#endif extern Process *erts_code_purger; @@ -1109,7 +1101,6 @@ void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); typedef struct { Eterm delay_time; int context_reds; - int input_reds; } ErtsModifiedTimings; extern Export *erts_delay_trap; @@ -1127,18 +1118,12 @@ extern ErtsModifiedTimings erts_modified_timings[]; extern int erts_no_line_info; extern Eterm erts_error_logger_warnings; extern int erts_initialized; -#if defined(USE_THREADS) && !defined(ERTS_SMP) -extern erts_tid_t erts_main_thread; -#endif extern int erts_compat_rel; extern int erts_use_sender_punish; void erl_start(int, char**); void erts_usage(void); Eterm erts_preloaded(Process* p); -#ifndef ERTS_SMP -extern void *erts_scheduler_stack_limit; -#endif /* erl_md5.c */ @@ -1182,8 +1167,9 @@ void erts_emergency_close_ports(void); void erts_ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon); Eterm erts_driver_monitor_to_ref(Eterm* hp, const ErlDrvMonitor *mon); -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT) -void erts_lcnt_enable_io_lock_count(int enable); +#if defined(ERTS_ENABLE_LOCK_COUNT) +void erts_lcnt_update_driver_locks(int enable); +void erts_lcnt_update_port_locks(int enable); #endif /* driver_tab.c */ @@ -1252,7 +1238,7 @@ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ; -Sint erts_native_filename_need(Eterm ioterm, int encoding); +Sint erts_native_filename_need(Eterm ioterm, int encoding, int allow_null); void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left); @@ -1287,7 +1273,8 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); void bin_write(fmtfn_t, void*, byte*, size_t); Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ -Sint erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len); +int erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written); +Sint erts_unicode_list_to_buf_len(Eterm list); struct Sint_buf { #if defined(ARCH_64) @@ -1392,7 +1379,7 @@ Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); -int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); +int erts_hibernate(Process* c_p, Eterm* reg); ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index a1f6f54543..93d1111904 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -98,7 +98,7 @@ index_put_entry(IndexTable* t, void* tmpl) * Do a write barrier here to allow readers to do lock free iteration. * erts_index_num_entries() does matching read barrier. */ - ERTS_SMP_WRITE_MEMORY_BARRIER; + ERTS_THR_WRITE_MEMORY_BARRIER; t->entries++; return p; diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 6c07571df6..30bc6a1121 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -88,7 +88,7 @@ ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t) * on tables where entries are never erased. * index_put_entry() does matching write barrier. */ - ERTS_SMP_READ_MEMORY_BARRIER; + ERTS_THR_READ_MEMORY_BARRIER; return ret; } diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab new file mode 100644 index 0000000000..07cc4bd527 --- /dev/null +++ b/erts/emulator/beam/instrs.tab @@ -0,0 +1,922 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +// Stack manipulation instructions + +allocate(NeedStack, Live) { + $AH($NeedStack, 0, $Live); +} + +allocate_heap(NeedStack, NeedHeap, Live) { + $AH($NeedStack, $NeedHeap, $Live); +} + +allocate_init(NeedStack, Live, Y) { + $AH($NeedStack, 0, $Live); + make_blank($Y); +} + +allocate_zero(NeedStack, Live) { + Eterm* ptr; + int i = $NeedStack; + $AH(i, 0, $Live); + for (ptr = E + i; ptr > E; ptr--) { + make_blank(*ptr); + } +} + +allocate_heap_zero(NeedStack, NeedHeap, Live) { + Eterm* ptr; + int i = $NeedStack; + $AH(i, $NeedHeap, $Live); + for (ptr = E + i; ptr > E; ptr--) { + make_blank(*ptr); + } +} + +// This instruction is probably never used (because it is combined with a +// a return). However, a future compiler might for some reason emit a +// deallocate not followed by a return, and that should work. + +deallocate(Deallocate) { + //| -no_prefetch + SET_CP(c_p, (BeamInstr *) cp_val(*E)); + E = ADD_BYTE_OFFSET(E, $Deallocate); +} + +deallocate_return(Deallocate) { + //| -no_next + int words_to_pop = $Deallocate; + SET_I((BeamInstr *) cp_val(*E)); + E = ADD_BYTE_OFFSET(E, words_to_pop); + CHECK_TERM(x(0)); + DispatchReturn; +} + +move_deallocate_return(Src, Deallocate) { + x(0) = $Src; + $deallocate_return($Deallocate); +} + +// Call instructions + +DISPATCH_REL(CallDest) { + //| -no_next + $SET_I_REL($CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + Dispatch(); +} + +DISPATCH_ABS(CallDest) { + //| -no_next + SET_I((BeamInstr *) $CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + Dispatch(); +} + +i_call(CallDest) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_REL($CallDest); +} + +move_call(Src, CallDest) { + x(0) = $Src; + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_REL($CallDest); +} + +i_call_last(CallDest, Deallocate) { + $deallocate($Deallocate); + $DISPATCH_REL($CallDest); +} + +move_call_last(Src, CallDest, Deallocate) { + x(0) = $Src; + $i_call_last($CallDest, $Deallocate); +} + +i_call_only(CallDest) { + $DISPATCH_REL($CallDest); +} + +move_call_only(Src, CallDest) { + x(0) = $Src; + $i_call_only($CallDest); +} + +DISPATCHX(Dest) { + //| -no_next + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest); + // Dispatchx assumes the Export* is in Arg(0) + I = (&$Dest) - 1; + Dispatchx(); +} + +i_call_ext(Dest) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCHX($Dest); +} + +i_move_call_ext(Src, Dest) { + x(0) = $Src; + $i_call_ext($Dest); +} + +i_call_ext_only(Dest) { + $DISPATCHX($Dest); +} + +i_move_call_ext_only(Dest, Src) { + x(0) = $Src; + $i_call_ext_only($Dest); +} + +i_call_ext_last(Dest, Deallocate) { + $deallocate($Deallocate); + $DISPATCHX($Dest); +} + +i_move_call_ext_last(Dest, StackOffset, Src) { + x(0) = $Src; + $i_call_ext_last($Dest, $StackOffset); +} + +APPLY(I, Deallocate, Next) { + //| -no_next + HEAVY_SWAPOUT; + $Next = apply(c_p, reg, $I, $Deallocate); + HEAVY_SWAPIN; +} + +HANDLE_APPLY_ERROR() { + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); + goto post_error_handling; +} + +i_apply() { + BeamInstr *next; + $APPLY(NULL, 0, next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_ABS(next); + } + $HANDLE_APPLY_ERROR(); +} + +i_apply_last(Deallocate) { + BeamInstr *next; + $APPLY(I, $Deallocate, next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_ABS(next); + } + $HANDLE_APPLY_ERROR(); +} + +i_apply_only() { + BeamInstr *next; + $APPLY(I, 0, next); + if (ERTS_LIKELY(next != NULL)) { + $DISPATCH_ABS(next); + } + $HANDLE_APPLY_ERROR(); +} + +FIXED_APPLY(Arity, I, Deallocate, Next) { + //| -no_next + HEAVY_SWAPOUT; + $Next = fixed_apply(c_p, reg, $Arity, $I, $Deallocate); + HEAVY_SWAPIN; +} + +apply(Arity) { + BeamInstr *next; + $FIXED_APPLY($Arity, NULL, 0, next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_ABS(next); + } + $HANDLE_APPLY_ERROR(); +} + +apply_last(Arity, Deallocate) { + BeamInstr *next; + $FIXED_APPLY($Arity, I, $Deallocate, next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_ABS(next); + } + $HANDLE_APPLY_ERROR(); +} + +APPLY_FUN(Next) { + HEAVY_SWAPOUT; + $Next = apply_fun(c_p, r(0), x(1), reg); + HEAVY_SWAPIN; +} + +HANDLE_APPLY_FUN_ERROR() { + goto find_func_info; +} + +DISPATCH_FUN(I) { + SET_I($I); + Dispatchfun(); +} + +i_apply_fun() { + BeamInstr *next; + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_FUN(next); + } + $HANDLE_APPLY_FUN_ERROR(); +} + +i_apply_fun_last(Deallocate) { + BeamInstr *next; + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_FUN(next); + } + $HANDLE_APPLY_FUN_ERROR(); +} + +i_apply_fun_only() { + BeamInstr *next; + $APPLY_FUN(next); + if (ERTS_LIKELY(next != NULL)) { + $DISPATCH_FUN(next); + } + $HANDLE_APPLY_FUN_ERROR(); +} + +CALL_FUN(Fun, Next) { + //| -no_next + HEAVY_SWAPOUT; + $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE); + HEAVY_SWAPIN; +} + +i_call_fun(Fun) { + BeamInstr *next; + $CALL_FUN($Fun, next); + if (ERTS_LIKELY(next != NULL)) { + SET_CP(c_p, $NEXT_INSTRUCTION); + $DISPATCH_FUN(next); + } + $HANDLE_APPLY_FUN_ERROR(); +} + +i_call_fun_last(Fun, Deallocate) { + BeamInstr *next; + $CALL_FUN($Fun, next); + if (ERTS_LIKELY(next != NULL)) { + $deallocate($Deallocate); + $DISPATCH_FUN(next); + } + $HANDLE_APPLY_FUN_ERROR(); +} + +return() { + SET_I(c_p->cp); + DTRACE_RETURN_FROM_PC(c_p); + + /* + * We must clear the CP to make sure that a stale value do not + * create a false module dependcy preventing code upgrading. + * It also means that we can use the CP in stack backtraces. + */ + c_p->cp = 0; + CHECK_TERM(r(0)); + HEAP_SPACE_VERIFIED(0); + DispatchReturn; +} + +get_list(Src, Hd, Tl) { + Eterm* tmp_ptr = list_val($Src); + Eterm hd, tl; + hd = CAR(tmp_ptr); + tl = CDR(tmp_ptr); + $Hd = hd; + $Tl = tl; +} + +i_get(Src, Dst) { + $Dst = erts_pd_hash_get(c_p, $Src); +} + +i_get_hash(Src, Hash, Dst) { + $Dst = erts_pd_hash_get_with_hx(c_p, $Hash, $Src); +} + +i_get_tuple_element(Src, Element, Dst) { + Eterm* src = ADD_BYTE_OFFSET(tuple_val($Src), $Element); + $Dst = *src; +} + +i_get_tuple_element2(Src, Element, Dst) { + Eterm* src; + Eterm* dst; + Eterm E1, E2; + src = ADD_BYTE_OFFSET(tuple_val($Src), $Element); + dst = &($Dst); + E1 = src[0]; + E2 = src[1]; + dst[0] = E1; + dst[1] = E2; +} + +i_get_tuple_element2y(Src, Element, D1, D2) { + Eterm* src; + Eterm E1, E2; + src = ADD_BYTE_OFFSET(tuple_val($Src), $Element); + E1 = src[0]; + E2 = src[1]; + $D1 = E1; + $D2 = E2; +} + +i_get_tuple_element3(Src, Element, Dst) { + Eterm* src; + Eterm* dst; + Eterm E1, E2, E3; + src = ADD_BYTE_OFFSET(tuple_val($Src), $Element); + dst = &($Dst); + E1 = src[0]; + E2 = src[1]; + E3 = src[2]; + dst[0] = E1; + dst[1] = E2; + dst[2] = E3; +} + +i_element := element_group.fetch.execute; + + +element_group.head() { + Eterm element_tuple; +} + +element_group.fetch(Src) { + element_tuple = $Src; +} + +element_group.execute(Fail, Index, Dst) { + Eterm element_index = $Index; + if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) { + Eterm* tp = tuple_val(element_tuple); + + if ((signed_val(element_index) >= 1) && + (signed_val(element_index) <= arityval(*tp))) { + $Dst = tp[signed_val(element_index)]; + $NEXT0(); + } + } + c_p->freason = BADARG; + $BIF_ERROR_ARITY_2($Fail, BIF_element_2, element_index, element_tuple); +} + +i_fast_element := fast_element_group.fetch.execute; + +fast_element_group.head() { + Eterm fast_element_tuple; +} + +fast_element_group.fetch(Src) { + fast_element_tuple = $Src; +} + +fast_element_group.execute(Fail, Index, Dst) { + if (ERTS_LIKELY(is_tuple(fast_element_tuple))) { + Eterm* tp = tuple_val(fast_element_tuple); + Eterm pos = $Index; /* Untagged integer >= 1 */ + if (pos <= arityval(*tp)) { + $Dst = tp[pos]; + $NEXT0(); + } + } + c_p->freason = BADARG; + $BIF_ERROR_ARITY_2($Fail, BIF_element_2, make_small($Index), fast_element_tuple); +} + +init(Y) { + make_blank($Y); +} + +init2(Y1, Y2) { + make_blank($Y1); + make_blank($Y2); +} + +init3(Y1, Y2, Y3) { + make_blank($Y1); + make_blank($Y2); + make_blank($Y3); +} + +i_make_fun(FunP, NumFree) { + HEAVY_SWAPOUT; + x(0) = new_fun(c_p, reg, (ErlFunEntry *) $FunP, $NumFree); + HEAVY_SWAPIN; +} + +i_trim(Words) { + Uint cp = E[0]; + E += $Words; + E[0] = cp; +} + +move(Src, Dst) { + $Dst = $Src; +} + +move3(S1, D1, S2, D2, S3, D3) { + $D1 = $S1; + $D2 = $S2; + $D3 = $S3; +} + +move_dup(Src, D1, D2) { + $D1 = $D2 = $Src; +} + +move2_par(S1, D1, S2, D2) { + Eterm V1, V2; + V1 = $S1; + V2 = $S2; + $D1 = V1; + $D2 = V2; +} + +move_shift(Src, SD, D) { + Eterm V; + V = $Src; + $D = $SD; + $SD = V; +} + +move_window3(S1, S2, S3, D) { + Eterm xt0, xt1, xt2; + Eterm* y = &$D; + xt0 = $S1; + xt1 = $S2; + xt2 = $S3; + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; +} + +move_window4(S1, S2, S3, S4, D) { + Eterm xt0, xt1, xt2, xt3; + Eterm* y = &$D; + xt0 = $S1; + xt1 = $S2; + xt2 = $S3; + xt3 = $S4; + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; +} + +move_window5(S1, S2, S3, S4, S5, D) { + Eterm xt0, xt1, xt2, xt3, xt4; + Eterm *y = &$D; + xt0 = $S1; + xt1 = $S2; + xt2 = $S3; + xt3 = $S4; + xt4 = $S5; + y[0] = xt0; + y[1] = xt1; + y[2] = xt2; + y[3] = xt3; + y[4] = xt4; +} + +move_return(Src) { + //| -no_next + x(0) = $Src; + SET_I(c_p->cp); + c_p->cp = 0; + DispatchReturn; +} + +move_x1(Src) { + x(1) = $Src; +} + +move_x2(Src) { + x(2) = $Src; +} + +node(Dst) { + $Dst = erts_this_node->sysname; +} + +put_list(Hd, Tl, Dst) { + HTOP[0] = $Hd; + HTOP[1] = $Tl; + $Dst = make_list(HTOP); + HTOP += 2; +} + +i_put_tuple := i_put_tuple.make.fill; + +i_put_tuple.make(Dst) { + $Dst = make_tuple(HTOP); +} + +i_put_tuple.fill(Arity) { + Eterm* hp = HTOP; + Eterm arity = $Arity; + + //| -no_next + *hp++ = make_arityval(arity); + I = $NEXT_INSTRUCTION; + do { + Eterm term = *I++; + switch (loader_tag(term)) { + case LOADER_X_REG: + *hp++ = x(loader_x_reg_index(term)); + break; + case LOADER_Y_REG: + *hp++ = y(loader_y_reg_index(term)); + break; + default: + *hp++ = term; + break; + } + } while (--arity != 0); + HTOP = hp; + ASSERT(VALID_INSTR(* (Eterm *)I)); + Goto(*I); +} + +self(Dst) { + $Dst = c_p->common.id; +} + +set_tuple_element(Element, Tuple, Offset) { + Eterm* p; + + ASSERT(is_tuple($Tuple)); + p = (Eterm *) ((unsigned char *) tuple_val($Tuple) + $Offset); + *p = $Element; +} + +swap(R1, R2) { + Eterm V = $R1; + $R1 = $R2; + $R2 = V; +} + +swap_temp(R1, R2, Tmp) { + Eterm V = $R1; + $R1 = $R2; + $R2 = $Tmp = V; +} + +test_heap(Nh, Live) { + $GC_TEST(0, $Nh, $Live); +} + +test_heap_1_put_list(Nh, Reg) { + $test_heap($Nh, 1); + $put_list($Reg, x(0), x(0)); +} + +is_integer_allocate(Fail, Src, NeedStack, Live) { + //| -no_prefetch + $is_integer($Fail, $Src); + $AH($NeedStack, 0, $Live); +} + +is_nonempty_list(Fail, Src) { + //| -no_prefetch + if (is_not_list($Src)) { + $FAIL($Fail); + } +} + +is_nonempty_list_test_heap(Fail, Need, Live) { + //| -no_prefetch + $is_nonempty_list($Fail, x(0)); + $test_heap($Need, $Live); +} + +is_nonempty_list_allocate(Fail, Src, Need, Live) { + //| -no_prefetch + $is_nonempty_list($Fail, $Src); + $AH($Need, 0, $Live); +} + +is_nonempty_list_get_list(Fail, Src, Hd, Tl) { + //| -no_prefetch + $is_nonempty_list($Fail, $Src); + $get_list($Src, $Hd, $Tl); +} + +jump(Fail) { + $JUMP($Fail); +} + +move_jump(Fail, Src) { + x(0) = $Src; + $jump($Fail); +} + +// +// Test instructions. +// + +is_atom(Fail, Src) { + if (is_not_atom($Src)) { + $FAIL($Fail); + } +} + +is_boolean(Fail, Src) { + if (($Src) != am_true && ($Src) != am_false) { + $FAIL($Fail); + } +} + +is_binary(Fail, Src) { + if (is_not_binary($Src) || binary_bitsize($Src) != 0) { + $FAIL($Fail); + } +} + +is_bitstring(Fail, Src) { + if (is_not_binary($Src)) { + $FAIL($Fail); + } +} + +is_float(Fail, Src) { + if (is_not_float($Src)) { + $FAIL($Fail); + } +} + +is_function(Fail, Src) { + if ( !(is_any_fun($Src)) ) { + $FAIL($Fail); + } +} + +is_function2(Fail, Fun, Arity) { + if (erl_is_function(c_p, $Fun, $Arity) != am_true ) { + $FAIL($Fail); + } +} + +is_integer(Fail, Src) { + if (is_not_integer($Src)) { + $FAIL($Fail); + } +} + +is_list(Fail, Src) { + if (is_not_list($Src) && is_not_nil($Src)) { + $FAIL($Fail); + } +} + +is_map(Fail, Src) { + if (is_not_map($Src)) { + $FAIL($Fail); + } +} + +is_nil(Fail, Src) { + if (is_not_nil($Src)) { + $FAIL($Fail); + } +} + +is_number(Fail, Src) { + if (is_not_integer($Src) && is_not_float($Src)) { + $FAIL($Fail); + } +} + +is_pid(Fail, Src) { + if (is_not_pid($Src)) { + $FAIL($Fail); + } +} + +is_port(Fail, Src) { + if (is_not_port($Src)) { + $FAIL($Fail); + } +} + +is_reference(Fail, Src) { + if (is_not_ref($Src)) { + $FAIL($Fail); + } +} + +is_tagged_tuple(Fail, Src, Arityval, Tag) { + Eterm term = $Src; + if (!(BEAM_IS_TUPLE(term) && + (tuple_val(term))[0] == $Arityval && + (tuple_val(term))[1] == $Tag)) { + $FAIL($Fail); + } +} + +is_tuple(Fail, Src) { + if (is_not_tuple($Src)) { + $FAIL($Fail); + } +} + +is_tuple_of_arity(Fail, Src, Arityval) { + Eterm term = $Src; + if (!(BEAM_IS_TUPLE(term) && *tuple_val(term) == $Arityval)) { + $FAIL($Fail); + } +} + +test_arity(Fail, Pointer, Arity) { + if (*tuple_val($Pointer) != $Arity) { + $FAIL($Fail); + } +} + +i_is_eq_exact_immed(Fail, X, Y) { + if ($X != $Y) { + $FAIL($Fail); + } +} + +i_is_ne_exact_immed(Fail, X, Y) { + if ($X == $Y) { + $FAIL($Fail); + } +} + +is_eq_exact(Fail, X, Y) { + if (!EQ($X, $Y)) { + $FAIL($Fail); + } +} + +i_is_eq_exact_literal(Fail, Src, Literal) { + if (!eq($Src, $Literal)) { + $FAIL($Fail); + } +} + +is_ne_exact(Fail, X, Y) { + if (EQ($X, $Y)) { + $FAIL($Fail); + } +} + +i_is_ne_exact_literal(Fail, Src, Literal) { + if (eq($Src, $Literal)) { + $FAIL($Fail); + } +} + +is_eq(Fail, X, Y) { + CMP_EQ_ACTION($X, $Y, $FAIL($Fail)); +} + +is_ne(Fail, X, Y) { + CMP_NE_ACTION($X, $Y, $FAIL($Fail)); +} + +is_lt(Fail, X, Y) { + CMP_LT_ACTION($X, $Y, $FAIL($Fail)); +} + +is_ge(Fail, X, Y) { + CMP_GE_ACTION($X, $Y, $FAIL($Fail)); +} + +badarg(Fail) { + $BADARG($Fail); + //| -no_next; +} + +badmatch(Src) { + c_p->fvalue = $Src; + c_p->freason = BADMATCH; + goto find_func_info; + //| -no_next; +} + +case_end(Src) { + c_p->fvalue = $Src; + c_p->freason = EXC_CASE_CLAUSE; + goto find_func_info; + //| -no_next; +} + +if_end() { + c_p->freason = EXC_IF_CLAUSE; + goto find_func_info; + //| -no_next; +} + +system_limit(Fail) { + $SYSTEM_LIMIT($Fail); + //| -no_next; +} + +catch(Y, Fail) { + c_p->catches++; + $Y = $Fail; +} + +catch_end(Y) { + c_p->catches--; + make_blank($Y); + if (is_non_value(r(0))) { + c_p->fvalue = NIL; + if (x(1) == am_throw) { + r(0) = x(2); + } else { + if (x(1) == am_error) { + SWAPOUT; + x(2) = add_stacktrace(c_p, x(2), x(3)); + SWAPIN; + } + /* only x(2) is included in the rootset here */ + if (E - HTOP < 3) { + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + } + r(0) = TUPLE2(HTOP, am_EXIT, x(2)); + HTOP += 3; + } + } + CHECK_TERM(r(0)); +} + +try_end(Y) { + c_p->catches--; + make_blank($Y); + if (is_non_value(r(0))) { + c_p->fvalue = NIL; + r(0) = x(1); + x(1) = x(2); + x(2) = x(3); + } +} + +try_case_end(Src) { + c_p->fvalue = $Src; + c_p->freason = EXC_TRY_CLAUSE; + goto find_func_info; + //| -no_next; +} + +i_raise() { + Eterm raise_trace = x(2); + Eterm raise_value = x(1); + struct StackTrace *s; + + c_p->fvalue = raise_value; + c_p->ftrace = raise_trace; + s = get_trace_from_exc(raise_trace); + if (s == NULL) { + c_p->freason = EXC_ERROR; + } else { + c_p->freason = PRIMARY_EXCEPTION(s->freason); + } + goto find_func_info; + //| -no_next +} + diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index d25e53ada0..85013af3ad 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -52,6 +52,7 @@ #include "erl_bif_unique.h" #include "erl_hl_timer.h" #include "erl_time.h" +#include "erl_io_queue.h" extern ErlDrvEntry fd_driver_entry; extern ErlDrvEntry vanilla_driver_entry; @@ -62,10 +63,10 @@ extern ErlDrvEntry forker_driver_entry; extern ErlDrvEntry *driver_tab[]; /* table of static drivers, only used during initialization */ erts_driver_t *driver_list; /* List of all drivers, static and dynamic. */ -erts_smp_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */ -static erts_smp_tsd_key_t driver_list_lock_status_key; /*stop recursive locks when calling +erts_rwmtx_t erts_driver_list_lock; /* Mutex for driver list */ +static erts_tsd_key_t driver_list_lock_status_key; /*stop recursive locks when calling driver init */ -static erts_smp_tsd_key_t driver_list_last_error_key; /* Save last DDLL error on a +static erts_tsd_key_t driver_list_last_error_key; /* Save last DDLL error on a per thread basis (for BC interfaces) */ ErtsPTab erts_port erts_align_attribute(ERTS_CACHE_LINE_SIZE); /* The port table */ @@ -93,22 +94,16 @@ static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *); static void terminate_port(Port *p); static void pdl_init(void); static int driver_failure_term(ErlDrvPort ix, Eterm term, int eof); -#ifdef ERTS_SMP static void driver_monitor_lock_pdl(Port *p); static void driver_monitor_unlock_pdl(Port *p); #define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 1) #define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port) #define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port) -#else -#define DRV_MONITOR_LOOKUP_PORT_LOCK_PDL(Port) erts_thr_drvport2port((Port), 0) -#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */ -#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */ -#endif #define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT) #define SMALL_WRITE_VEC 16 -static ERTS_INLINE ErlIOQueue* +static ERTS_INLINE ErlPortIOQueue* drvport2ioq(ErlDrvPort drvport) { Port *prt = erts_thr_drvport2port(drvport, 0); @@ -121,13 +116,13 @@ static ERTS_INLINE int is_port_ioq_empty(Port *pp) { int res; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); if (!pp->port_data_lock) - res = (pp->ioq.size == 0); + res = (erts_ioq_size(&pp->ioq) == 0); else { ErlDrvPDL pdl = pp->port_data_lock; erts_mtx_lock(&pdl->mtx); - res = (pp->ioq.size == 0); + res = (erts_ioq_size(&pp->ioq) == 0); erts_mtx_unlock(&pdl->mtx); } return res; @@ -142,14 +137,14 @@ erts_is_port_ioq_empty(Port *pp) Uint erts_port_ioq_size(Port *pp) { - int res; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ErlDrvSizeT res; + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); if (!pp->port_data_lock) - res = pp->ioq.size; + res = erts_ioq_size(&pp->ioq); else { ErlDrvPDL pdl = pp->port_data_lock; erts_mtx_lock(&pdl->mtx); - res = pp->ioq.size; + res = erts_ioq_size(&pp->ioq); erts_mtx_unlock(&pdl->mtx); } return (Uint) res; @@ -211,14 +206,13 @@ dtrace_drvport_str(ErlDrvPort drvport, char *port_buf) static ERTS_INLINE void kill_port(Port *pp) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(pp)); ERTS_TRACER_CLEAR(&ERTS_TRACER(pp)); erts_ptab_delete_element(&erts_port, &pp->common); /* Time of death */ erts_port_task_free_port(pp); /* In non-smp case the port structure may have been deallocated now */ } -#ifdef ERTS_SMP #ifdef ERTS_ENABLE_LOCK_CHECK int @@ -226,12 +220,11 @@ erts_lc_is_port_locked(Port *prt) { if (!prt) return 0; - ERTS_SMP_LC_ASSERT(prt->lock); - return erts_smp_lc_mtx_is_locked(prt->lock); + ERTS_LC_ASSERT(prt->lock); + return erts_lc_mtx_is_locked(prt->lock); } #endif -#endif /* #ifdef ERTS_SMP */ static void initq(Port* prt); @@ -255,32 +248,21 @@ static ERTS_INLINE void port_init_instr(Port *prt * Stuff that need to be initialized with the port id * in the instrumented case, but not in the normal case. */ -#ifdef ERTS_SMP ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { - char *lock_str = "port_lock"; -#ifdef ERTS_ENABLE_LOCK_COUNT - Uint16 opt = ((erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) - ? 0 : ERTS_LCNT_LT_DISABLE); -#else - Uint16 opt = 0; -#endif - erts_mtx_init_locked_x_opt(prt->lock, lock_str, id, opt); + erts_mtx_init_locked(prt->lock, "port_lock", id, ERTS_LOCK_FLAGS_CATEGORY_IO); } -#endif erts_port_task_init_sched(&prt->sched, id); } #if !ERTS_PORT_INIT_INSTR_NEED_ID static ERTS_INLINE void port_init_instr_abort(Port *prt) { -#ifdef ERTS_SMP ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { erts_mtx_unlock(prt->lock); erts_mtx_destroy(prt->lock); } -#endif erts_port_task_fini_sched(&prt->sched); } #endif @@ -316,7 +298,6 @@ static Port *create_port(char *name, erts_aint32_t state = ERTS_PORT_SFLG_CONNECTED; erts_aint32_t x_pts_flgs = 0; -#ifdef ERTS_SMP ErtsRunQueue *runq; if (!driver_lock) { /* Align size for mutex following port struct */ @@ -324,7 +305,6 @@ static Port *create_port(char *name, size += sizeof(erts_mtx_t); } else -#endif port_size = size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); #ifdef DEBUG @@ -358,7 +338,6 @@ static Port *create_port(char *name, p += busy_port_queue_size; } -#ifdef ERTS_SMP if (driver_lock) { prt->lock = driver_lock; erts_mtx_lock(driver_lock); @@ -372,13 +351,9 @@ static Port *create_port(char *name, runq = erts_get_runq_current(NULL); else runq = ERTS_RUNQ_IX(0); - erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); + erts_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq); prt->xports = NULL; -#else - erts_atomic32_init_nob(&prt->refc, 1); - prt->cleanup = 0; -#endif erts_port_task_pre_init_sched(&prt->sched, busy_port_queue); @@ -399,7 +374,7 @@ static Port *create_port(char *name, prt->common.u.alive.reg = NULL; ERTS_PTMR_INIT(prt); erts_port_task_handle_init(&prt->timeout_task); - erts_smp_atomic_init_nob(&prt->psd, (erts_aint_t) NULL); + erts_atomic_init_nob(&prt->psd, (erts_aint_t) NULL); prt->async_open_port = NULL; prt->drv_data = (SWord) 0; prt->os_pid = -1; @@ -426,10 +401,8 @@ static Port *create_port(char *name, #if !ERTS_PORT_INIT_INSTR_NEED_ID port_init_instr_abort(prt); #endif -#ifdef ERTS_SMP if (driver_lock) erts_mtx_unlock(driver_lock); -#endif if (enop) *enop = 0; erts_free(ERTS_ALC_T_PORT, prt); @@ -442,7 +415,7 @@ static Port *create_port(char *name, initq(prt); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (erts_port_schedule_all_ops) x_pts_flgs |= ERTS_PTS_FLG_FORCE_SCHED; @@ -451,29 +424,17 @@ static Port *create_port(char *name, x_pts_flgs |= ERTS_PTS_FLG_PARALLELISM; if (x_pts_flgs) - erts_smp_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs); + erts_atomic32_read_bor_nob(&prt->sched.flags, x_pts_flgs); erts_atomic32_set_relb(&prt->state, state); return prt; } -#ifndef ERTS_SMP -void -erts_port_cleanup(Port *prt) -{ - if (prt->drv_ptr && prt->drv_ptr->handle) - erts_ddll_dereference_driver(prt->drv_ptr->handle); - prt->drv_ptr = NULL; - erts_port_dec_refc(prt); -} -#endif void erts_port_free(Port *prt) { -#if defined(ERTS_SMP) || defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK) erts_aint32_t state = erts_atomic32_read_nob(&prt->state); -#endif ERTS_LC_ASSERT(state & (ERTS_PORT_SFLG_INITIALIZING | ERTS_PORT_SFLG_FREE)); ASSERT(state & ERTS_PORT_SFLG_PORT_DEBUG); @@ -487,7 +448,6 @@ erts_port_free(Port *prt) prt->async_open_port = NULL; } -#ifdef ERTS_SMP ASSERT(prt->lock); if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) erts_mtx_destroy(prt->lock); @@ -504,7 +464,6 @@ erts_port_free(Port *prt) */ if (prt->drv_ptr->handle) erts_ddll_dereference_driver(prt->drv_ptr->handle); -#endif erts_free(ERTS_ALC_T_PORT, prt); } @@ -515,41 +474,17 @@ erts_port_free(Port *prt) */ static void initq(Port* prt) { - ErlIOQueue* q = &prt->ioq; - ERTS_LC_ASSERT(!prt->port_data_lock); - - q->size = 0; - q->v_head = q->v_tail = q->v_start = q->v_small; - q->v_end = q->v_small + SMALL_IO_QUEUE; - q->b_head = q->b_tail = q->b_start = q->b_small; - q->b_end = q->b_small + SMALL_IO_QUEUE; + erts_ioq_init(&prt->ioq, ERTS_ALC_T_IOQ, 1); } static void stopq(Port* prt) { - ErlIOQueue* q; - ErlDrvBinary** binp; if (prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); - q = &prt->ioq; - binp = q->b_head; - - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - - while(binp < q->b_tail) { - if (*binp != NULL) - driver_free_binary(*binp); - binp++; - } - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->v_start = q->v_end = q->v_head = q->v_tail = NULL; - q->b_start = q->b_end = q->b_head = q->b_tail = NULL; - q->size = 0; + erts_ioq_clear(&prt->ioq); if (prt->port_data_lock) { driver_pdl_unlock(prt->port_data_lock); @@ -563,7 +498,7 @@ erts_save_suspend_process_on_port(Port *prt, Process *process) int saved; erts_aint32_t flags; erts_port_task_sched_lock(&prt->sched); - flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + flags = erts_atomic32_read_nob(&prt->sched.flags); saved = (flags & ERTS_PTS_FLGS_BUSY) && !(flags & ERTS_PTS_FLG_EXIT); if (saved) erts_proclist_store_last(&prt->suspended, erts_proclist_create(process)); @@ -607,16 +542,16 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ erts_mtx_t *driver_lock = NULL; int cprt_flgs = 0; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); if (!driver) { for (driver = driver_list; driver; driver = driver->next) { if (sys_strcmp(driver->name, name) == 0) break; } if (!driver) { - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG); } } @@ -661,19 +596,17 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ } if (driver == NULL || (driver != &spawn_driver && opts->exit_status)) { - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); ERTS_OPEN_DRIVER_RET(NULL, -3, BADARG); } -#ifdef ERTS_SMP driver_lock = driver->lock; -#endif if (driver->handle != NULL) { erts_ddll_increment_port_count(driver->handle); erts_ddll_reference_driver(driver->handle); } - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); /* * We'll set up the port before calling the start function, @@ -686,9 +619,9 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ port = create_port(name, driver, driver_lock, cprt_flgs, pid, &port_errno); if (!port) { if (driver->handle) { - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); erts_ddll_dereference_driver(driver->handle); } if (port_errno) @@ -756,11 +689,9 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(port, am_out, am_open); } -#ifdef ERTS_SMP if (port->xports) erts_port_handle_xports(port); ASSERT(!port->xports); -#endif } if (error_type) { @@ -775,9 +706,9 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ port->linebuf = NULL; } if (driver->handle != NULL) { - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); } kill_port(port); erts_port_release(port); @@ -789,7 +720,6 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ #undef ERTS_OPEN_DRIVER_RET } -#ifdef ERTS_SMP struct ErtsXPortsList_ { ErtsXPortsList *next; @@ -798,7 +728,6 @@ struct ErtsXPortsList_ { ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(xports_list, ErtsXPortsList, 50, ERTS_ALC_T_XPORTS_LIST) -#endif /* * Driver function to create new instances of a driver @@ -818,7 +747,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ Process *rp; erts_mtx_t *driver_lock = NULL; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; /* Need to be called from a scheduler thread */ if (!erts_get_scheduler_id()) @@ -832,12 +761,12 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ if (!rp) return ERTS_INVALID_ERL_DRV_PORT; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(creator_port)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(creator_port)); driver = creator_port->drv_ptr; - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); if (!erts_ddll_driver_ok(driver->handle)) { - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); return ERTS_INVALID_ERL_DRV_PORT; } @@ -846,35 +775,33 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ erts_ddll_reference_referenced_driver(driver->handle); } -#ifdef ERTS_SMP driver_lock = driver->lock; -#endif - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); /* Inherit parallelism flag from parent */ if (ERTS_PTS_FLG_PARALLELISM & - erts_smp_atomic32_read_nob(&creator_port->sched.flags)) + erts_atomic32_read_nob(&creator_port->sched.flags)) cprt_flgs |= ERTS_CREATE_PORT_FLAG_PARALLELISM; port = create_port(name, driver, driver_lock, cprt_flgs, pid, NULL); if (!port) { if (driver->handle) { - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); erts_ddll_dereference_driver(driver->handle); } return ERTS_INVALID_ERL_DRV_PORT; } - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(port)); - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (ERTS_PROC_IS_EXITING(rp)) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (driver->handle) { - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(driver->handle); - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); } kill_port(port); erts_port_release(port); @@ -883,23 +810,20 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */ erts_add_link(&ERTS_P_LINKS(port), LINK_PID, pid); erts_add_link(&ERTS_P_LINKS(rp), LINK_PID, port->common.id); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); -#ifdef ERTS_SMP if (!driver_lock) { ErtsXPortsList *xplp = xports_list_alloc(); xplp->port = port; xplp->next = creator_port->xports; creator_port->xports = xplp; } -#endif port->drv_data = (UWord) drv_data; return ERTS_Port2ErlDrvPort(port); } -#ifdef ERTS_SMP int erts_port_handle_xports(Port *prt) { int reds = 0; @@ -928,312 +852,6 @@ int erts_port_handle_xports(Port *prt) prt->xports = NULL; return reds; } -#endif - -/* Fills a possibly deep list of chars and binaries into vec -** Small characters are first stored in the buffer buf of length ln -** binaries found are copied and linked into msoh -** Return vector length on succsess, -** -1 on overflow -** -2 on type error -*/ - -#ifdef DEBUG -#define MAX_SYSIOVEC_IOVLEN (1ull << (32 - 1)) -#else -#define MAX_SYSIOVEC_IOVLEN (1ull << (sizeof(((SysIOVec*)0)->iov_len) * 8 - 1)) -#endif - -static ERTS_INLINE void -io_list_to_vec_set_vec(SysIOVec **iov, ErlDrvBinary ***binv, - ErlDrvBinary *bin, byte *ptr, Uint len, - int *vlen) -{ - while (len > MAX_SYSIOVEC_IOVLEN) { - (*iov)->iov_base = ptr; - (*iov)->iov_len = MAX_SYSIOVEC_IOVLEN; - ptr += MAX_SYSIOVEC_IOVLEN; - len -= MAX_SYSIOVEC_IOVLEN; - (*iov)++; - (*vlen)++; - *(*binv)++ = bin; - } - (*iov)->iov_base = ptr; - (*iov)->iov_len = len; - *(*binv)++ = bin; - (*iov)++; - (*vlen)++; -} - -static int -io_list_to_vec(Eterm obj, /* io-list */ - SysIOVec* iov, /* io vector */ - ErlDrvBinary** binv, /* binary reference vector */ - ErlDrvBinary* cbin, /* binary to store characters */ - ErlDrvSizeT bin_limit) /* small binaries limit */ -{ - DECLARE_ESTACK(s); - Eterm* objp; - byte *buf = (byte*)cbin->orig_bytes; - Uint len = cbin->orig_size; - Uint csize = 0; - int vlen = 0; - byte* cptr = buf; - - goto L_jump_start; /* avoid push */ - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_jump_start: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - if (is_byte(obj)) { - if (len == 0) - goto L_overflow; - *buf++ = unsigned_val(obj); - csize++; - len--; - } else if (is_binary(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto handle_binary; - } else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } else if (!is_nil(obj)) { - goto L_type_error; - } - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { - goto handle_binary; - } else if (!is_nil(obj)) { - goto L_type_error; - } - } else if (is_binary(obj)) { - Eterm real_bin; - Uint offset; - Eterm* bptr; - ErlDrvSizeT size; - int bitoffs; - int bitsize; - - handle_binary: - size = binary_size(obj); - ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); - ASSERT(bitsize == 0); - bptr = binary_val(real_bin); - if (*bptr == HEADER_PROC_BIN) { - ProcBin* pb = (ProcBin *) bptr; - if (bitoffs != 0) { - if (len < size) { - goto L_overflow; - } - erts_copy_bits(pb->bytes+offset, bitoffs, 1, - (byte *) buf, 0, 1, size*8); - csize += size; - buf += size; - len -= size; - } else if (bin_limit && size < bin_limit) { - if (len < size) { - goto L_overflow; - } - sys_memcpy(buf, pb->bytes+offset, size); - csize += size; - buf += size; - len -= size; - } else { - if (csize != 0) { - io_list_to_vec_set_vec(&iov, &binv, cbin, - cptr, csize, &vlen); - cptr = buf; - csize = 0; - } - if (pb->flags) { - erts_emasculate_writable_binary(pb); - } - io_list_to_vec_set_vec( - &iov, &binv, Binary2ErlDrvBinary(pb->val), - pb->bytes+offset, size, &vlen); - } - } else { - ErlHeapBin* hb = (ErlHeapBin *) bptr; - if (len < size) { - goto L_overflow; - } - copy_binary_to_buffer(buf, 0, - ((byte *) hb->data)+offset, bitoffs, - 8*size); - csize += size; - buf += size; - len -= size; - } - } else if (!is_nil(obj)) { - goto L_type_error; - } - } - - if (csize != 0) { - io_list_to_vec_set_vec(&iov, &binv, cbin, cptr, csize, &vlen); - } - - DESTROY_ESTACK(s); - return vlen; - - L_type_error: - DESTROY_ESTACK(s); - return -2; - - L_overflow: - DESTROY_ESTACK(s); - return -1; -} - -#define IO_LIST_VEC_COUNT(obj) \ -do { \ - Uint _size = binary_size(obj); \ - Eterm _real; \ - ERTS_DECLARE_DUMMY(Uint _offset); \ - int _bitoffs; \ - int _bitsize; \ - ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize); \ - if (_bitsize != 0) goto L_type_error; \ - if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG && \ - _bitoffs == 0) { \ - b_size += _size; \ - if (b_size < _size) goto L_overflow_error; \ - in_clist = 0; \ - v_size++; \ - /* If iov_len is smaller then Uint we split the binary into*/ \ - /* multiple smaller (2GB) elements in the iolist.*/ \ - v_size += _size / MAX_SYSIOVEC_IOVLEN; \ - if (_size >= ERL_SMALL_IO_BIN_LIMIT) { \ - p_in_clist = 0; \ - p_v_size++; \ - } else { \ - p_c_size += _size; \ - if (!p_in_clist) { \ - p_in_clist = 1; \ - p_v_size++; \ - } \ - } \ - } else { \ - c_size += _size; \ - if (c_size < _size) goto L_overflow_error; \ - if (!in_clist) { \ - in_clist = 1; \ - v_size++; \ - } \ - p_c_size += _size; \ - if (!p_in_clist) { \ - p_in_clist = 1; \ - p_v_size++; \ - } \ - } \ -} while (0) - - -/* - * Returns 0 if successful and a non-zero value otherwise. - * - * Return values through pointers: - * *vsize - SysIOVec size needed for a writev - * *csize - Number of bytes not in binary (in the common binary) - * *pvsize - SysIOVec size needed if packing small binaries - * *pcsize - Number of bytes in the common binary if packing - * *total_size - Total size of iolist in bytes - */ - -static int -io_list_vec_len(Eterm obj, int* vsize, Uint* csize, - Uint* pvsize, Uint* pcsize, - ErlDrvSizeT* total_size) -{ - DECLARE_ESTACK(s); - Eterm* objp; - Uint v_size = 0; - Uint c_size = 0; - Uint b_size = 0; - Uint in_clist = 0; - Uint p_v_size = 0; - Uint p_c_size = 0; - Uint p_in_clist = 0; - Uint total; - - goto L_jump_start; /* avoid a push */ - - while (!ESTACK_ISEMPTY(s)) { - obj = ESTACK_POP(s); - L_jump_start: - if (is_list(obj)) { - L_iter_list: - objp = list_val(obj); - obj = CAR(objp); - - if (is_byte(obj)) { - c_size++; - if (c_size == 0) { - goto L_overflow_error; - } - if (!in_clist) { - in_clist = 1; - v_size++; - } - p_c_size++; - if (!p_in_clist) { - p_in_clist = 1; - p_v_size++; - } - } - else if (is_binary(obj)) { - IO_LIST_VEC_COUNT(obj); - } - else if (is_list(obj)) { - ESTACK_PUSH(s, CDR(objp)); - goto L_iter_list; /* on head */ - } - else if (!is_nil(obj)) { - goto L_type_error; - } - - obj = CDR(objp); - if (is_list(obj)) - goto L_iter_list; /* on tail */ - else if (is_binary(obj)) { /* binary tail is OK */ - IO_LIST_VEC_COUNT(obj); - } - else if (!is_nil(obj)) { - goto L_type_error; - } - } - else if (is_binary(obj)) { - IO_LIST_VEC_COUNT(obj); - } - else if (!is_nil(obj)) { - goto L_type_error; - } - } - - total = c_size + b_size; - if (total < c_size) { - goto L_overflow_error; - } - *total_size = (ErlDrvSizeT) total; - - DESTROY_ESTACK(s); - *vsize = v_size; - *csize = c_size; - *pvsize = p_v_size; - *pcsize = p_c_size; - return 0; - - L_type_error: - L_overflow_error: - DESTROY_ESTACK(s); - return 1; -} typedef enum { ERTS_TRY_IMM_DRV_CALL_OK, @@ -1280,12 +898,12 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) invalid_sched_flags |= ERTS_PTS_FLG_PARALLELISM; if (sp->pre_chk_sched_flags) { - sp->sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + sp->sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (sp->sched_flags & invalid_sched_flags) return ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS; } - if (erts_smp_port_trylock(prt) == EBUSY) + if (erts_port_trylock(prt) == EBUSY) return ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK; invalid_state = sp->state; @@ -1299,7 +917,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) if (prof_runnable_ports) erts_port_task_sched_lock(&prt->sched); - act = erts_smp_atomic32_read_nob(&prt->sched.flags); + act = erts_atomic32_read_nob(&prt->sched.flags); do { erts_aint32_t new; @@ -1311,7 +929,7 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) } exp = act; new = act | ERTS_PTS_FLG_EXEC_IMM; - act = erts_smp_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp); + act = erts_atomic32_cmpxchg_mb(&prt->sched.flags, new, exp); } while (act != exp); sp->sched_flags = act; @@ -1333,14 +951,14 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp) profile_runnable_proc(c_p, am_inactive); reds_left_in = ERTS_BIF_REDS_LEFT(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); } ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS); sp->reds_left_in = reds_left_in; prt->reds = CONTEXT_REDS - reds_left_in; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { if (prof_runnable_ports && !(act & (ERTS_PTS_FLG_IN_RUNQ|ERTS_PTS_FLG_EXEC))) @@ -1378,9 +996,9 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) if (prof_runnable_ports) erts_port_task_sched_lock(&prt->sched); - act = erts_smp_atomic32_read_band_mb(&prt->sched.flags, + act = erts_atomic32_read_band_mb(&prt->sched.flags, ~ERTS_PTS_FLG_EXEC_IMM); - ERTS_SMP_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM); + ERTS_LC_ASSERT(act & ERTS_PTS_FLG_EXEC_IMM); if (prof_runnable_ports | IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) { if (IS_TRACED_FL(prt, F_TRACE_SCHED_PORTS)) @@ -1395,7 +1013,7 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp) erts_port_release(prt); if (c_p) { - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (reds != (CONTEXT_REDS - sp->reds_left_in)) { int bump_reds = reds - (CONTEXT_REDS - sp->reds_left_in); @@ -1506,7 +1124,7 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg, Port* prt) prt); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } @@ -1524,7 +1142,7 @@ erts_schedule_proc2port_signal(Process *c_p, int sched_res; if (!refp) { if (c_p) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); } else { ASSERT(c_p); @@ -1545,20 +1163,20 @@ erts_schedule_proc2port_signal(Process *c_p, * otherwise, next receive will *not* work * as expected! */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); if (ERTS_PROC_PENDING_EXIT(c_p)) { /* need to exit caller instead */ - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); KILL_CATCHES(c_p); c_p->freason = EXC_EXIT; return ERTS_PORT_OP_CALLER_EXIT; } - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); c_p->msg.save = c_p->msg.last; - erts_smp_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE + erts_proc_unlock(c_p, (ERTS_PROC_LOCKS_MSG_RECEIVE | ERTS_PROC_LOCK_MAIN)); } @@ -1574,7 +1192,7 @@ erts_schedule_proc2port_signal(Process *c_p, task_flags); if (c_p) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); if (sched_res != 0) { if (refp) { @@ -1585,9 +1203,9 @@ erts_schedule_proc2port_signal(Process *c_p, * containing the reference created above... */ ASSERT(c_p); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); JOIN_MESSAGE(c_p); - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); *refp = NIL; } return ERTS_PORT_OP_DROPPED; @@ -1620,14 +1238,14 @@ send_badsig(Port *prt) { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; Process* rp; Eterm connected = ERTS_PORT_GET_CONNECTED(prt); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; ERTS_LC_ASSERT(erts_get_scheduler_id()); ASSERT(is_internal_pid(connected)); rp = erts_proc_lookup_raw(connected); if (rp) { - erts_smp_proc_lock(rp, rp_locks); + erts_proc_lock(rp, rp_locks); if (!ERTS_PROC_IS_EXITING(rp)) (void) erts_send_exit_signal(NULL, prt->common.id, @@ -1638,7 +1256,7 @@ send_badsig(Port *prt) { NULL, 0); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } /* exit sent */ } /* send_badsig */ @@ -1761,7 +1379,7 @@ call_driver_outputv(int bang_op, ErlDrvSizeT size = evp->size; ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); @@ -1804,8 +1422,7 @@ cleanup_scheduled_outputv(ErlIOVec *ev, ErlDrvBinary *cbinp) int i; /* Need to free all binaries */ for (i = 1; i < ev->vsize; i++) - if (ev->binv[i]) - driver_free_binary(ev->binv[i]); + driver_free_binary(ev->binv[i]); if (cbinp) driver_free_binary(cbinp); } @@ -1819,7 +1436,7 @@ port_sig_outputv(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *s case ERTS_PROC2PORT_SIG_EXEC: /* Execution of a scheduled outputv() call */ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) reply = am_badarg; @@ -1875,7 +1492,7 @@ call_driver_output(int bang_op, else { ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt) + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt) || ERTS_IS_CRASH_DUMPING); #ifdef USE_VM_PROBES @@ -1926,7 +1543,7 @@ port_sig_output(Port *prt, erts_aint32_t state, int op, ErtsProc2PortSigData *si case ERTS_PROC2PORT_SIG_EXEC: /* Execution of a scheduled output() call */ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) reply = am_badarg; @@ -1968,21 +1585,19 @@ int erts_port_output_async(Port *prt, Eterm from, Eterm list) { - ErtsPortOpResult res; ErtsProc2PortSigData *sigdp; erts_driver_t *drv = prt->drv_ptr; size_t size; int task_flags; ErtsProc2PortSigCallback port_sig_callback; - ErlDrvBinary *cbin = NULL; - ErlIOVec *evp = NULL; + ErtsIOQBinary *cbin = NULL; + ErtsIOVec *evp = NULL; char *buf = NULL; ErtsPortTaskHandle *ns_pthp; if (drv->outputv) { - ErlIOVec ev; SysIOVec* ivp; - ErlDrvBinary** bvp; + ErtsIOQBinary** bvp; int vsize; Uint csize; Uint pvsize; @@ -1992,91 +1607,63 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list) char *ptr; int i; - Eterm* bptr = NULL; - Uint offset; - - if (is_binary(list)) { - /* We optimize for when we get a procbin without offset */ - Eterm real_bin; - int bitoffs; - int bitsize; - ERTS_GET_REAL_BIN(list, real_bin, offset, bitoffs, bitsize); - bptr = binary_val(real_bin); - if (*bptr == HEADER_PROC_BIN && bitoffs == 0) { - size = binary_size(list); - vsize = 1; - } else - bptr = NULL; - } - - if (!bptr) { - if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) - goto bad_value; + if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize, + &size, ERL_SMALL_IO_BIN_LIMIT)) + goto bad_value; - /* To pack or not to pack (small binaries) ...? */ - if (vsize >= SMALL_WRITE_VEC) { - /* Do pack */ - vsize = pvsize + 1; - csize = pcsize; - blimit = ERL_SMALL_IO_BIN_LIMIT; - } - cbin = driver_alloc_binary(csize); + /* To pack or not to pack (small binaries) ...? */ + if (vsize >= SMALL_WRITE_VEC) { + /* Do pack */ + vsize = pvsize + 1; + csize = pcsize; + blimit = ERL_SMALL_IO_BIN_LIMIT; + } + if (csize) { + cbin = (ErtsIOQBinary *)driver_alloc_binary(csize); if (!cbin) erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); } - iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); binv_offset = iov_offset; binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); alloc_size = binv_offset; - alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *); sigdp = erts_port_task_alloc_p2p_sig_data_extra(alloc_size, (void**)&ptr); - evp = (ErlIOVec *) ptr; - ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); - bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + evp = (ErtsIOVec *) ptr; + ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset); ivp[0].iov_base = NULL; ivp[0].iov_len = 0; bvp[0] = NULL; - if (bptr) { - ProcBin* pb = (ProcBin *) bptr; - - ivp[1].iov_base = pb->bytes+offset; - ivp[1].iov_len = size; - bvp[1] = Binary2ErlDrvBinary(pb->val); - - evp->vsize = 1; - } else { - - evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); - if (evp->vsize < 0) { - if (evp != &ev) - erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); - driver_free_binary(cbin); - goto bad_value; - } + evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1, cbin, + blimit, 1); + if (evp->driver.vsize < 0) { + erts_free(ERTS_ALC_T_DRV_CMD_DATA, evp); + driver_free_binary(&cbin->driver); + goto bad_value; } #if 0 /* This assertion may say something useful, but it can be falsified during the emulator test suites. */ ASSERT(evp->vsize == vsize); #endif - evp->vsize++; - evp->size = size; /* total size */ + evp->driver.vsize++; + evp->driver.size = size; /* total size */ /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) + for (i = 1; i < evp->driver.vsize; i++) if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + driver_binary_inc_refc(&bvp[i]->driver); sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; - sigdp->u.outputv.evp = evp; - sigdp->u.outputv.cbinp = cbin; + sigdp->u.outputv.evp = &evp->driver; + sigdp->u.outputv.cbinp = &cbin->driver; port_sig_callback = port_sig_outputv; } else { ErlDrvSizeT ERTS_DECLARE_DUMMY(r); @@ -2102,26 +1689,18 @@ erts_port_output_async(Port *prt, Eterm from, Eterm list) sigdp->u.output.size = size; port_sig_callback = port_sig_output; } - sigdp->flags = 0; ns_pthp = NULL; task_flags = 0; - res = erts_schedule_proc2port_signal(NULL, - prt, - ERTS_INVALID_PID, - NULL, - sigdp, - task_flags, - ns_pthp, - port_sig_callback); + erts_schedule_proc2port_signal(NULL, + prt, + ERTS_INVALID_PID, + NULL, + sigdp, + task_flags, + ns_pthp, + port_sig_callback); - if (res != ERTS_PORT_OP_SCHEDULED) { - if (drv->outputv) - cleanup_scheduled_outputv(evp, cbin); - else - cleanup_scheduled_output(buf); - return 1; - } return 1; bad_value: @@ -2155,8 +1734,8 @@ erts_port_output(Process *c_p, erts_aint32_t sched_flags, busy_flgs, invalid_flags; int task_flags; ErtsProc2PortSigCallback port_sig_callback; - ErlDrvBinary *cbin = NULL; - ErlIOVec *evp = NULL; + ErtsIOQBinary *cbin = NULL; + ErtsIOVec *evp = NULL; char *buf = NULL; int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL); int async_nosuspend; @@ -2179,7 +1758,7 @@ erts_port_output(Process *c_p, * Assumes caller have checked that port is valid... */ - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT)) return ((sched_flags & ERTS_PTS_FLG_EXIT) ? ERTS_PORT_OP_DROPPED @@ -2202,11 +1781,11 @@ erts_port_output(Process *c_p, } #endif if (drv->outputv) { - ErlIOVec ev; + ErtsIOVec ev; SysIOVec iv[SMALL_WRITE_VEC]; - ErlDrvBinary* bv[SMALL_WRITE_VEC]; + ErtsIOQBinary* bv[SMALL_WRITE_VEC]; SysIOVec* ivp; - ErlDrvBinary** bvp; + ErtsIOQBinary** bvp; int vsize; Uint csize; Uint pvsize; @@ -2214,18 +1793,19 @@ erts_port_output(Process *c_p, Uint blimit; size_t iov_offset, binv_offset, alloc_size; - if (io_list_vec_len(list, &vsize, &csize, &pvsize, &pcsize, &size)) + if (erts_ioq_iodata_vec_len(list, &vsize, &csize, &pvsize, &pcsize, + &size, ERL_SMALL_IO_BIN_LIMIT)) goto bad_value; iov_offset = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErlIOVec)); binv_offset = iov_offset; binv_offset += ERTS_ALC_DATA_ALIGN_SIZE((vsize+1)*sizeof(SysIOVec)); alloc_size = binv_offset; - alloc_size += (vsize+1)*sizeof(ErlDrvBinary *); + alloc_size += (vsize+1)*sizeof(ErtsIOQBinary *); if (try_call && vsize < SMALL_WRITE_VEC) { - ivp = ev.iov = iv; - bvp = ev.binv = bv; + ivp = ev.common.iov = iv; + bvp = ev.common.binv = bv; evp = &ev; } else { @@ -2236,9 +1816,9 @@ erts_port_output(Process *c_p, sigdp = erts_port_task_alloc_p2p_sig_data_extra( alloc_size, (void**)&ptr); } - evp = (ErlIOVec *) ptr; - ivp = evp->iov = (SysIOVec *) (ptr + iov_offset); - bvp = evp->binv = (ErlDrvBinary **) (ptr + binv_offset); + evp = (ErtsIOVec *) ptr; + ivp = evp->driver.iov = (SysIOVec *) (ptr + iov_offset); + bvp = evp->common.binv = (ErtsIOQBinary **) (ptr + binv_offset); } /* To pack or not to pack (small binaries) ...? */ @@ -2254,23 +1834,26 @@ erts_port_output(Process *c_p, } /* Use vsize and csize from now on */ - cbin = driver_alloc_binary(csize); - if (!cbin) - erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + if (csize) { + cbin = (ErtsIOQBinary *)driver_alloc_binary(csize); + if (!cbin) + erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, ERTS_SIZEOF_Binary(csize)); + } /* Element 0 is for driver usage to add header block */ ivp[0].iov_base = NULL; ivp[0].iov_len = 0; bvp[0] = NULL; - evp->vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit); - if (evp->vsize < 0) { + evp->driver.vsize = erts_ioq_iodata_to_vec(list, ivp+1, bvp+1, + cbin, blimit, 1); + if (evp->driver.vsize < 0) { if (evp != &ev) { if (try_call) erts_free(ERTS_ALC_T_TMP, evp); else erts_port_task_free_p2p_sig_data(sigdp); } - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); goto bad_value; } #if 0 @@ -2278,19 +1861,19 @@ erts_port_output(Process *c_p, be falsified during the emulator test suites. */ ASSERT(evp->vsize == vsize); #endif - evp->vsize++; - evp->size = size; /* total size */ + evp->driver.vsize++; + evp->driver.size = size; /* total size */ if (!try_call) { int i; /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) - if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + for (i = 1; i < evp->driver.vsize; i++) + if (bvp[i]) + driver_binary_inc_refc(&bvp[i]->driver); } else { int i; - ErlIOVec *new_evp; + ErtsIOVec *new_evp; ErtsTryImmDrvCallResult try_call_res; ErtsTryImmDrvCallState try_call_state = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( @@ -2313,14 +1896,14 @@ erts_port_output(Process *c_p, from, prt, drv, - evp); + &evp->driver); if (force_immediate_call) finalize_force_imm_drv_call(&try_call_state); else finalize_imm_drv_call(&try_call_state); /* Fall through... */ case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); if (evp != &ev) { ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); @@ -2334,7 +1917,7 @@ erts_port_output(Process *c_p, sched_flags = try_call_state.sched_flags; if (async_nosuspend && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { - driver_free_binary(cbin); + driver_free_binary(&cbin->driver); if (evp != &ev) { ASSERT(!sigdp); erts_free(ERTS_ALC_T_TMP, evp); @@ -2349,9 +1932,9 @@ erts_port_output(Process *c_p, } /* Need to increase refc on all binaries */ - for (i = 1; i < evp->vsize; i++) + for (i = 1; i < evp->driver.vsize; i++) if (bvp[i]) - driver_binary_inc_refc(bvp[i]); + driver_binary_inc_refc(&bvp[i]->driver); /* The port task and iovec is allocated in the same structure as an optimization. This @@ -2364,18 +1947,18 @@ erts_port_output(Process *c_p, if (evp != &ev) { /* Copy from TMP alloc to port task */ sys_memcpy((void *) new_evp, (void *) evp, alloc_size); - new_evp->iov = (SysIOVec *) (((char *) new_evp) - + iov_offset); - bvp = new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) - + binv_offset); + new_evp->driver.iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + bvp = new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp) + + binv_offset); #ifdef DEBUG - ASSERT(new_evp->vsize == evp->vsize); - ASSERT(new_evp->size == evp->size); - for (i = 0; i < evp->vsize; i++) { - ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); - ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); - ASSERT(new_evp->binv[i] == evp->binv[i]); + ASSERT(new_evp->driver.vsize == evp->driver.vsize); + ASSERT(new_evp->driver.size == evp->driver.size); + for (i = 0; i < evp->driver.vsize; i++) { + ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len); + ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base); + ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]); } #endif @@ -2384,24 +1967,24 @@ erts_port_output(Process *c_p, else { /* from stack allocated structure; offsets may differ */ sys_memcpy((void *) new_evp, (void *) evp, sizeof(ErlIOVec)); - new_evp->iov = (SysIOVec *) (((char *) new_evp) - + iov_offset); - sys_memcpy((void *) new_evp->iov, - (void *) evp->iov, - evp->vsize * sizeof(SysIOVec)); - new_evp->binv = (ErlDrvBinary **) (((char *) new_evp) - + binv_offset); - sys_memcpy((void *) new_evp->binv, - (void *) evp->binv, - evp->vsize * sizeof(ErlDrvBinary *)); + new_evp->driver.iov = (SysIOVec *) (((char *) new_evp) + + iov_offset); + sys_memcpy((void *) new_evp->driver.iov, + (void *) evp->driver.iov, + evp->driver.vsize * sizeof(SysIOVec)); + new_evp->common.binv = (ErtsIOQBinary **) (((char *) new_evp) + + binv_offset); + sys_memcpy((void *) new_evp->common.binv, + (void *) evp->common.binv, + evp->driver.vsize * sizeof(ErtsIOQBinary *)); #ifdef DEBUG - ASSERT(new_evp->vsize == evp->vsize); - ASSERT(new_evp->size == evp->size); - for (i = 0; i < evp->vsize; i++) { - ASSERT(new_evp->iov[i].iov_len == evp->iov[i].iov_len); - ASSERT(new_evp->iov[i].iov_base == evp->iov[i].iov_base); - ASSERT(new_evp->binv[i] == evp->binv[i]); + ASSERT(new_evp->driver.vsize == evp->driver.vsize); + ASSERT(new_evp->driver.size == evp->driver.size); + for (i = 0; i < evp->driver.vsize; i++) { + ASSERT(new_evp->driver.iov[i].iov_len == evp->driver.iov[i].iov_len); + ASSERT(new_evp->driver.iov[i].iov_base == evp->driver.iov[i].iov_base); + ASSERT(new_evp->driver.binv[i] == evp->driver.binv[i]); } #endif @@ -2412,8 +1995,8 @@ erts_port_output(Process *c_p, sigdp->flags = ERTS_P2P_SIG_TYPE_OUTPUTV; sigdp->u.outputv.from = from; - sigdp->u.outputv.evp = evp; - sigdp->u.outputv.cbinp = cbin; + sigdp->u.outputv.evp = &evp->driver; + sigdp->u.outputv.cbinp = &cbin->driver; port_sig_callback = port_sig_outputv; } else { @@ -2554,15 +2137,11 @@ erts_port_output(Process *c_p, port_sig_callback); if (res != ERTS_PORT_OP_SCHEDULED) { - if (drv->outputv) - cleanup_scheduled_outputv(evp, cbin); - else - cleanup_scheduled_output(buf); return res; } if (!(flags & ERTS_PORT_SIG_FLG_FORCE)) { - sched_flags = erts_smp_atomic32_read_acqb(&prt->sched.flags); + sched_flags = erts_atomic32_read_acqb(&prt->sched.flags); if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)) { if (async_nosuspend) erts_port_task_tmp_handle_detach(ns_pthp); @@ -2736,21 +2315,14 @@ erts_port_exit(Process *c_p, &bp->off_heap); } - res = erts_schedule_proc2port_signal(c_p, - prt, - c_p ? c_p->common.id : from, - refp, - sigdp, - 0, - NULL, - port_sig_exit); - - if (res == ERTS_PORT_OP_DROPPED) { - if (bp) - free_message_buffer(bp); - } - - return res; + return erts_schedule_proc2port_signal(c_p, + prt, + c_p ? c_p->common.id : from, + refp, + sigdp, + 0, + NULL, + port_sig_exit); } static ErtsPortOpResult @@ -2794,9 +2366,9 @@ set_port_connected(int bang_op, Process *rp = erts_proc_lookup_raw(connect); if (!rp) return ERTS_PORT_OP_DROPPED; - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); if (ERTS_PROC_IS_EXITING(rp)) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); return ERTS_PORT_OP_DROPPED; } @@ -2808,7 +2380,7 @@ set_port_connected(int bang_op, ERTS_PORT_SET_CONNECTED(prt, connect); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (IS_TRACED_FL(prt, F_TRACE_PORTS)) trace_port(prt, am_getting_linked, connect); @@ -3001,7 +2573,7 @@ port_link_failure(Eterm port_id, Eterm linker) trace_proc(NULL, 0, rp, am_getting_unlinked, port_id); } if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } } @@ -3087,7 +2659,7 @@ port_monitor_failure(Eterm port_id, Eterm origin, Eterm ref_DOWN) * caller has never seen it yet. */ erts_queue_monitor_message(origin_p, &p_locks, ref_DOWN, am_port, port_id, am_noproc); - erts_smp_proc_unlock(origin_p, p_locks); + erts_proc_unlock(origin_p, p_locks); } /* Origin wants to monitor port Prt. State contains possible error, which has @@ -3115,7 +2687,7 @@ port_monitor(Port *prt, erts_aint32_t state, Eterm origin, erts_add_monitor(&ERTS_P_MONITORS(prt), MON_TARGET, ref, origin, name_or_nil); - erts_smp_proc_unlock(origin_p, p_locks); + erts_proc_unlock(origin_p, p_locks); } else { failure: port_monitor_failure(prt->common.id, origin, ref); @@ -3206,7 +2778,7 @@ port_demonitor_failure(Eterm port_id, Eterm origin, Eterm ref) erts_destroy_monitor(mon1); } - erts_smp_proc_unlock(origin_p, rp_locks); + erts_proc_unlock(origin_p, rp_locks); } /* Origin wants to demonitor port Prt. State contains possible error, which has @@ -3236,7 +2808,7 @@ port_demonitor(Port *port, erts_aint32_t state, Eterm origin, Eterm ref) } } if (origin_p) { /* when origin is dying, it won't be found */ - erts_smp_proc_unlock(origin_p, p_locks); + erts_proc_unlock(origin_p, p_locks); } } else { port_demonitor_failure(port->common.id, origin, ref); @@ -3323,10 +2895,10 @@ init_ack_send_reply(Port *port, Eterm resp) if (!is_internal_port(resp)) { Process *rp = erts_proc_lookup_raw(port->async_open_port->to); - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_lock(rp, ERTS_PROC_LOCK_LINK); erts_remove_link(&ERTS_P_LINKS(port), port->async_open_port->to); erts_remove_link(&ERTS_P_LINKS(rp), port->common.id); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); } port_sched_op_reply(port->async_open_port->to, port->async_open_port->ref, @@ -3390,9 +2962,9 @@ void erts_init_io(int port_tab_size, { ErlDrvEntry** dp; UWord common_element_size; - erts_smp_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - drv_list_rwmtx_opts.type = ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; - drv_list_rwmtx_opts.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_rwmtx_opt_t drv_list_rwmtx_opts = ERTS_RWMTX_OPT_DEFAULT_INITER; + drv_list_rwmtx_opts.type = ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ; + drv_list_rwmtx_opts.lived = ERTS_RWMTX_LONG_LIVED; erts_atomic64_init_nob(&bytes_in, 0); erts_atomic64_init_nob(&bytes_out, 0); @@ -3400,11 +2972,9 @@ void erts_init_io(int port_tab_size, common_element_size = ERTS_ALC_DATA_ALIGN_SIZE(sizeof(Port)); common_element_size += ERTS_ALC_DATA_ALIGN_SIZE(sizeof(ErtsPortTaskBusyPortQ)); common_element_size += 10; /* name */ -#ifdef ERTS_SMP common_element_size += sizeof(erts_mtx_t); init_xports_list_alloc(); -#endif pdl_init(); @@ -3419,13 +2989,12 @@ void erts_init_io(int port_tab_size, else if (port_tab_size < ERTS_MIN_PORTS) port_tab_size = ERTS_MIN_PORTS; - erts_smp_rwmtx_init_opt(&erts_driver_list_lock, - &drv_list_rwmtx_opts, - "driver_list"); + erts_rwmtx_init_opt(&erts_driver_list_lock, &drv_list_rwmtx_opts, "driver_list", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); driver_list = NULL; - erts_smp_tsd_key_create(&driver_list_lock_status_key, + erts_tsd_key_create(&driver_list_lock_status_key, "erts_driver_list_lock_status_key"); - erts_smp_tsd_key_create(&driver_list_last_error_key, + erts_tsd_key_create(&driver_list_last_error_key, "erts_driver_list_last_error_key"); erts_ptab_init_table(&erts_port, @@ -3440,8 +3009,8 @@ void erts_init_io(int port_tab_size, sys_init_io(); - erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1); - erts_smp_rwmtx_rwlock(&erts_driver_list_lock); + erts_tsd_set(driver_list_lock_status_key, (void *) 1); + erts_rwmtx_rwlock(&erts_driver_list_lock); init_driver(&fd_driver, &fd_driver_entry, NULL); init_driver(&vanilla_driver, &vanilla_driver_entry, NULL); @@ -3453,73 +3022,101 @@ void erts_init_io(int port_tab_size, for (dp = driver_tab; *dp != NULL; dp++) erts_add_driver_entry(*dp, NULL, 1); - erts_smp_tsd_set(driver_list_lock_status_key, NULL); - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_tsd_set(driver_list_lock_status_key, NULL); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } -#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) -static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) +#if defined(ERTS_ENABLE_LOCK_COUNT) +static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { - if (enable) - erts_lcnt_init_lock_x(&dp->lock->lcnt, - "driver_lock", - ERTS_LCNT_LT_MUTEX, - erts_atom_put((byte*)dp->name, - sys_strlen(dp->name), - ERTS_ATOM_ENC_LATIN1, - 1)); - - else - erts_lcnt_destroy_lock(&dp->lock->lcnt); + if (enable) { + Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name), + ERTS_ATOM_ENC_LATIN1, 1); + erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", name_as_atom, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } else { + erts_lcnt_uninstall(&dp->lock->lcnt); + } } } -static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) +static void lcnt_enable_port_lock_count(Port *prt, int enable) { erts_aint32_t state = erts_atomic32_read_nob(&prt->state); - if (!enable) { - erts_lcnt_destroy_lock(&prt->sched.mtx.lcnt); - if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) - erts_lcnt_destroy_lock(&prt->lock->lcnt); + + if(enable) { + ErlDrvPDL pdl = prt->port_data_lock; + + erts_lcnt_install_new_lock_info(&prt->sched.mtx.lcnt, "port_sched_lock", + prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + + if(pdl) { + erts_lcnt_install_new_lock_info(&pdl->mtx.lcnt, "port_data_lock", + prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } + + if(state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + erts_lcnt_install_new_lock_info(&prt->lock->lcnt, "port_lock", + prt->common.id, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } + } else { + erts_lcnt_uninstall(&prt->sched.mtx.lcnt); + + if(prt->port_data_lock) { + erts_lcnt_uninstall(&prt->port_data_lock->mtx.lcnt); + } + + if(state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + erts_lcnt_uninstall(&prt->lock->lcnt); + } } - else { - erts_lcnt_init_lock_x(&prt->sched.mtx.lcnt, - "port_sched_lock", - ERTS_LCNT_LT_MUTEX, - prt->common.id); - if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) - erts_lcnt_init_lock_x(&prt->lock->lcnt, - "port_lock", - ERTS_LCNT_LT_MUTEX, - prt->common.id); +} + +void erts_lcnt_update_driver_locks(int enable) { + erts_driver_t *driver; + + lcnt_enable_driver_lock_count(&vanilla_driver, enable); + lcnt_enable_driver_lock_count(&spawn_driver, enable); +#ifndef __WIN32__ + lcnt_enable_driver_lock_count(&forker_driver, enable); +#endif + lcnt_enable_driver_lock_count(&fd_driver, enable); + + erts_rwmtx_rlock(&erts_driver_list_lock); + + for (driver = driver_list; driver; driver = driver->next) { + lcnt_enable_driver_lock_count(driver, enable); } + + erts_rwmtx_runlock(&erts_driver_list_lock); } -void erts_lcnt_enable_io_lock_count(int enable) { - erts_driver_t *dp; - int ix, max = erts_ptab_max(&erts_port); - Port *prt; +void erts_lcnt_update_port_locks(int enable) { + int i, max; - for (ix = 0; ix < max; ix++) { - if ((prt = erts_pix2port(ix)) != NULL) { - lcnt_enable_port_lock_count(prt, enable); + max = erts_ptab_max(&erts_port); + + for(i = 0; i < max; i++) { + int delay_handle; + Port *port; + + delay_handle = erts_thr_progress_unmanaged_delay(); + port = erts_pix2port(i); + + if(port != NULL) { + lcnt_enable_port_lock_count(port, enable); } - } /* for all ports */ - lcnt_enable_drv_lock_count(&vanilla_driver, enable); - lcnt_enable_drv_lock_count(&spawn_driver, enable); -#ifndef __WIN32__ - lcnt_enable_drv_lock_count(&forker_driver, enable); -#endif - lcnt_enable_drv_lock_count(&fd_driver, enable); - /* enable lock counting in all drivers */ - for (dp = driver_list; dp; dp = dp->next) { - lcnt_enable_drv_lock_count(dp, enable); + if(delay_handle != ERTS_THR_PRGR_DHANDLE_MANAGED) { + erts_thr_progress_unmanaged_continue(delay_handle); + } } -} /* enable/disable lock counting of ports */ -#endif /* defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) */ +} + +#endif /* defined(ERTS_ENABLE_LOCK_COUNT) */ + /* * Buffering of data when using line oriented I/O on ports */ @@ -3698,12 +3295,10 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res) ErtsProcLocks rp_locks = 0; int scheduler = erts_get_scheduler_id() != 0; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; ASSERT(!prt || prt->common.id == sender); -#ifdef ERTS_SMP - ASSERT(!prt || erts_lc_is_port_locked(prt)); -#endif + ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); ASSERT(is_internal_port(sender) && is_internal_pid(pid)); @@ -3731,7 +3326,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res) erts_queue_message(rp, rp_locks, mp, tuple, sender); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); @@ -3762,8 +3357,8 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, int scheduler = erts_get_scheduler_id() != 0; int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; need = 3 + 3 + 2*hlen; @@ -3830,7 +3425,7 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); } @@ -3865,7 +3460,7 @@ static void flush_linebuf_messages(Port *prt, erts_aint32_t state) LineBufContext lc; int ret; - ERTS_SMP_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(!prt || erts_lc_is_port_locked(prt)); if (!prt) return; @@ -3909,8 +3504,8 @@ deliver_vec_message(Port* prt, /* Port */ erts_aint32_t state; int trace_send = IS_TRACED_FL(prt, F_TRACE_SEND); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; /* * Check arguments for validity. @@ -4001,7 +3596,7 @@ deliver_vec_message(Port* prt, /* Port */ ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); } @@ -4036,8 +3631,8 @@ static void flush_port(Port *p) { int fpe_was_unmasked; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->flush != NULL) { ERTS_MSACC_PUSH_STATE_M(); @@ -4069,11 +3664,9 @@ static void flush_port(Port *p) if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) { trace_sched_ports_where(p, am_out, am_flush); } -#ifdef ERTS_SMP if (p->xports) erts_port_handle_xports(p); ASSERT(!p->xports); -#endif } if ((erts_atomic32_read_nob(&p->state) & ERTS_PORT_SFLGS_DEAD) == 0 && is_port_ioq_empty(p)) { @@ -4091,8 +3684,8 @@ terminate_port(Port *prt) erts_aint32_t state; ErtsPrtSD *psd; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(!ERTS_P_LINKS(prt)); ASSERT(!ERTS_P_MONITORS(prt)); @@ -4134,11 +3727,9 @@ terminate_port(Port *prt) (*drv->stop)((ErlDrvData)prt->drv_data); erts_unblock_fpe(fpe_was_unmasked); ERTS_MSACC_POP_STATE_M(); -#ifdef ERTS_SMP if (prt->xports) erts_port_handle_xports(prt); ASSERT(!prt->xports); -#endif } if (is_internal_port(send_closed_port_id) @@ -4146,9 +3737,9 @@ terminate_port(Port *prt) trace_port_send(prt, connected_id, am_closed, 1); if(drv->handle != NULL) { - erts_smp_rwmtx_rlock(&erts_driver_list_lock); + erts_rwmtx_rlock(&erts_driver_list_lock); erts_ddll_decrement_port_count(drv->handle); - erts_smp_rwmtx_runlock(&erts_driver_list_lock); + erts_rwmtx_runlock(&erts_driver_list_lock); } stopq(prt); /* clear queue memory */ if(prt->linebuf != NULL){ @@ -4158,7 +3749,7 @@ terminate_port(Port *prt) erts_cleanup_port_data(prt); - psd = (ErtsPrtSD *) erts_smp_atomic_read_nob(&prt->psd); + psd = (ErtsPrtSD *) erts_atomic_read_nob(&prt->psd); if (psd) erts_free(ERTS_ALC_T_PRTSD, psd); @@ -4171,7 +3762,7 @@ terminate_port(Port *prt) * port has been removed from the port table (in kill_port()). */ if ((state & ERTS_PORT_SFLG_HALT) - && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { + && (erts_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ erts_flush_async_exit(erts_halt_code, ""); } @@ -4199,7 +3790,7 @@ static void sweep_one_monitor(ErtsMonitor *mon, void *vpsc) goto done; } rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), mon->ref); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon == NULL) { goto done; } @@ -4273,7 +3864,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) 0); if (xres >= 0) { if (rp_locks & ERTS_PROC_LOCKS_XSIG_SEND) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); + erts_proc_unlock(rp, ERTS_PROC_LOCKS_XSIG_SEND); rp_locks &= ~ERTS_PROC_LOCKS_XSIG_SEND; } /* We didn't exit the process and it is traced */ @@ -4283,7 +3874,7 @@ static void sweep_one_link(ErtsLink *lnk, void *vpsc) erts_destroy_link(rlnk); } - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } erts_destroy_link(lnk); @@ -4318,7 +3909,7 @@ port_fire_one_monitor(ErtsMonitor *mon, void *ctx0) UnUseTmpHeapNoproc(3); rmon = erts_remove_monitor(&ERTS_P_MONITORS(origin), mon->ref); - erts_smp_proc_unlock(origin, origin_locks); + erts_proc_unlock(origin, origin_locks); if (rmon) { erts_destroy_monitor(rmon); @@ -4344,8 +3935,8 @@ erts_deliver_port_exit(Port *prt, Eterm from, Eterm reason, int send_closed, Eterm modified_reason; erts_aint32_t state, set_state_flags; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); modified_reason = (reason == am_kill) ? am_killed : reason; @@ -4706,7 +4297,7 @@ port_sig_control(Port *prt, prt); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); goto done; } } @@ -4759,7 +4350,7 @@ erts_port_control(Process* c_p, int copy; ErtsProc2PortSigData *sigdp; - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (sched_flags & ERTS_PTS_FLG_EXIT) return ERTS_PORT_OP_BADARG; @@ -4930,10 +4521,9 @@ erts_port_control(Process* c_p, 0, NULL, port_sig_control); - if (res != ERTS_PORT_OP_SCHEDULED) { - cleanup_scheduled_control(binp, bufp); + if (res != ERTS_PORT_OP_SCHEDULED) return ERTS_PORT_OP_BADARG; - } + return res; } @@ -5072,11 +4662,11 @@ port_sig_call(Port *prt, prt); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); goto done; } if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } } } @@ -5110,7 +4700,7 @@ erts_port_call(Process* c_p, erts_aint32_t sched_flags; ErtsProc2PortSigData *sigdp; - sched_flags = erts_smp_atomic32_read_nob(&prt->sched.flags); + sched_flags = erts_atomic32_read_nob(&prt->sched.flags); if (sched_flags & ERTS_PTS_FLG_EXIT) { return ERTS_PORT_OP_BADARG; } @@ -5223,10 +4813,9 @@ erts_port_call(Process* c_p, 0, NULL, port_sig_call); - if (res != ERTS_PORT_OP_SCHEDULED) { - cleanup_scheduled_call(bufp); + if (res != ERTS_PORT_OP_SCHEDULED) return ERTS_PORT_OP_BADARG; - } + return res; } @@ -5329,7 +4918,7 @@ port_sig_info(Port *prt, prt); } if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } return ERTS_PORT_REDS_INFO; } @@ -5398,7 +4987,7 @@ typedef struct { Uint sched_id; Eterm pid; Uint32 refn[ERTS_REF_NUMBERS]; - erts_smp_atomic32_t refc; + erts_atomic32_t refc; } ErtsIOBytesReq; static void @@ -5448,10 +5037,10 @@ reply_io_bytes(void *vreq) if (req->sched_id == sched_id) rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } - if (erts_smp_atomic32_dec_read_nob(&req->refc) == 0) + if (erts_atomic32_dec_read_nob(&req->refc) == 0) erts_free(ERTS_ALC_T_IOB_REQ, req); } @@ -5474,16 +5063,14 @@ erts_request_io_bytes(Process *c_p) req->refn[0] = refn[0]; req->refn[1] = refn[1]; req->refn[2] = refn[2]; - erts_smp_atomic32_init_nob(&req->refc, + erts_atomic32_init_nob(&req->refc, (erts_aint32_t) erts_no_schedulers); -#ifdef ERTS_SMP if (erts_no_schedulers > 1) erts_schedule_multi_misc_aux_work(1, erts_no_schedulers, reply_io_bytes, (void *) req); -#endif reply_io_bytes((void *) req); @@ -5568,14 +5155,14 @@ set_busy_port(ErlDrvPort dprt, int on) DTRACE_CHARBUF(port_str, 16); #endif - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; prt = erts_drvport2port(dprt); if (prt == ERTS_INVALID_ERL_DRV_PORT) return; if (on) { - flags = erts_smp_atomic32_read_bor_acqb(&prt->sched.flags, + flags = erts_atomic32_read_bor_acqb(&prt->sched.flags, ERTS_PTS_FLG_BUSY_PORT); if (flags & ERTS_PTS_FLG_BUSY_PORT) return; /* Already busy */ @@ -5591,7 +5178,7 @@ set_busy_port(ErlDrvPort dprt, int on) } #endif } else { - flags = erts_smp_atomic32_read_band_acqb(&prt->sched.flags, + flags = erts_atomic32_read_band_acqb(&prt->sched.flags, ~ERTS_PTS_FLG_BUSY_PORT); if (!(flags & ERTS_PTS_FLG_BUSY_PORT)) return; /* Already non-busy */ @@ -5685,7 +5272,7 @@ int get_port_flags(ErlDrvPort ix) if (prt == ERTS_INVALID_ERL_DRV_PORT) return 0; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); flags = 0; if (state & ERTS_PORT_SFLG_BINARY_IO) @@ -5701,8 +5288,8 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) int fpe_was_unmasked; ERTS_MSACC_PUSH_STATE_M(); - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(p)); if (len > (Uint) INT_MAX) erts_exit(ERTS_ABORT_EXIT, @@ -5731,10 +5318,10 @@ int async_ready(Port *p, void* data) { int need_free = 1; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (p) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(p)); if (p->drv_ptr->ready_async != NULL) { ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_PORT); #ifdef USE_VM_PROBES @@ -5803,24 +5390,17 @@ erts_stale_drv_select(Eterm port, switch (mode) { case ERL_DRV_READ | ERL_DRV_WRITE: type = "Input/Output"; - goto deselect; case ERL_DRV_WRITE: type = "Output"; - goto deselect; case ERL_DRV_READ: type = "Input"; - deselect: - if (deselect) { - driver_select(drv_port, hndl, - mode | ERL_DRV_USE_NO_CALLBACK, - 0); - } - break; default: - type = "Event"; - if (deselect) - driver_event(drv_port, hndl, NULL); - break; + type = ""; + } + if (deselect) { + driver_select(drv_port, hndl, + mode | ERL_DRV_USE_NO_CALLBACK, + 0); } dsbufp = erts_create_logger_dsbuf(); @@ -5919,8 +5499,8 @@ void driver_report_exit(ErlDrvPort ix, int status) if (prt == ERTS_INVALID_ERL_DRV_PORT) return; - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); pid = ERTS_PORT_GET_CONNECTED(prt); ASSERT(is_internal_pid(pid)); @@ -5943,7 +5523,7 @@ void driver_report_exit(ErlDrvPort ix, int status) ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); } @@ -6583,7 +6163,7 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) } if (rp) { if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); if (!scheduler) erts_proc_dec_refc(rp); } @@ -6598,9 +6178,7 @@ static ERTS_INLINE int deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p, Port **trace_prt) { -#ifdef ERTS_SMP ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); -#endif erts_aint32_t state; int res = 1; Port *prt = erts_port_lookup_raw((Eterm) port_id); @@ -6618,24 +6196,20 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p, goto done; } if (connected_p) { -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) ETHR_MEMBAR(ETHR_LoadLoad); -#endif *connected_p = ERTS_PORT_GET_CONNECTED(prt); } done: -#ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { - ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt)); erts_thr_progress_unmanaged_continue(dhndl); ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); } else -#endif if (res == 1) { - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); *trace_prt = prt; } return res; @@ -6663,13 +6237,13 @@ driver_output_term(ErlDrvPort drvport, ErlDrvTermData* data, int len) erts_aint32_t state; Port* prt; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; /* NOTE! It *not* safe to access 'drvport' from unmanaged threads. */ prt = erts_drvport2port_state(drvport, &state); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; /* invalid (dead) */ - ERTS_SMP_CHK_NO_PROC_LOCKS; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_CHK_NO_PROC_LOCKS; + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; @@ -6706,16 +6280,14 @@ driver_send_term(ErlDrvPort drvport, * internal data representation for ErlDrvPort. */ Port* prt = NULL; - ERTS_SMP_CHK_NO_PROC_LOCKS; -#ifdef ERTS_SMP + ERTS_CHK_NO_PROC_LOCKS; if (erts_thr_progress_is_managed_thread()) -#endif { erts_aint32_t state; prt = erts_drvport2port_state(drvport, &state); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; /* invalid (dead) */ - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; } @@ -6735,11 +6307,11 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, Port* prt = erts_drvport2port_state(ix, &state); ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; @@ -6749,6 +6321,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { + erts_atomic64_inc_nob(&prt->dist_entry->in); return erts_net_message(prt, prt->dist_entry, (byte*) hbuf, hlen, @@ -6774,12 +6347,12 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, Port* prt = erts_drvport2port_state(ix, &state); ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; @@ -6789,6 +6362,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, else erts_atomic64_add_nob(&bytes_in, (erts_aint64_t) (hlen + len)); if (state & ERTS_PORT_SFLG_DISTRIBUTION) { + erts_atomic64_inc_nob(&prt->dist_entry->in); if (len == 0) return erts_net_message(prt, prt->dist_entry, @@ -6813,7 +6387,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, int driver_output(ErlDrvPort ix, char* buf, ErlDrvSizeT len) { - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; return driver_output2(ix, NULL, 0, buf, len); } @@ -6829,7 +6403,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, erts_aint32_t state; ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; ASSERT(vec->size >= skip); if (vec->size <= skip) @@ -6840,7 +6414,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (state & ERTS_PORT_SFLG_CLOSING) return 0; @@ -7054,7 +6628,6 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl); } -#ifdef ERTS_SMP static void driver_monitor_lock_pdl(Port *p) { if (p->port_data_lock) { @@ -7063,7 +6636,7 @@ static void driver_monitor_lock_pdl(Port *p) { /* Now we either have the port lock or the port_data_lock */ ERTS_LC_ASSERT(!p->port_data_lock || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); - ERTS_SMP_LC_ASSERT(p->port_data_lock + ERTS_LC_ASSERT(p->port_data_lock || erts_lc_is_port_locked(p)); } @@ -7071,14 +6644,13 @@ static void driver_monitor_unlock_pdl(Port *p) { /* We should either have the port lock or the port_data_lock */ ERTS_LC_ASSERT(!p->port_data_lock || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx))); - ERTS_SMP_LC_ASSERT(p->port_data_lock + ERTS_LC_ASSERT(p->port_data_lock || erts_lc_is_port_locked(p)); if (p->port_data_lock) { driver_pdl_unlock(p->port_data_lock); } } -#endif /* * exported driver_pdl_* functions ... @@ -7093,7 +6665,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id); + erts_mtx_init(&pdl->mtx, "port_data_lock", pp->common.id, ERTS_LOCK_FLAGS_CATEGORY_IO); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -7157,307 +6729,51 @@ driver_pdl_dec_refc(ErlDrvPDL pdl) return refc; } -/* expand queue to hold n elements in tail or head */ -static int expandq(ErlIOQueue* q, int n, int tail) -/* tail: 0 if make room in head, make room in tail otherwise */ -{ - int h_sz; /* room before header */ - int t_sz; /* room after tail */ - int q_sz; /* occupied */ - int nvsz; - SysIOVec* niov; - ErlDrvBinary** nbinv; - - h_sz = q->v_head - q->v_start; - t_sz = q->v_end - q->v_tail; - q_sz = q->v_tail - q->v_head; - - if (tail && (n <= t_sz)) /* do we need to expand tail? */ - return 0; - else if (!tail && (n <= h_sz)) /* do we need to expand head? */ - return 0; - else if (n > (h_sz + t_sz)) { /* need to allocate */ - /* we may get little extra but it ok */ - nvsz = (q->v_end - q->v_start) + n; - - niov = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(SysIOVec)); - if (!niov) - return -1; - nbinv = erts_alloc_fnf(ERTS_ALC_T_IOQ, nvsz * sizeof(ErlDrvBinary**)); - if (!nbinv) { - erts_free(ERTS_ALC_T_IOQ, (void *) niov); - return -1; - } - if (tail) { - sys_memcpy(niov, q->v_head, q_sz*sizeof(SysIOVec)); - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - q->v_start = niov; - q->v_end = niov + nvsz; - q->v_head = q->v_start; - q->v_tail = q->v_head + q_sz; - - sys_memcpy(nbinv, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->b_start = nbinv; - q->b_end = nbinv + nvsz; - q->b_head = q->b_start; - q->b_tail = q->b_head + q_sz; - } - else { - sys_memcpy(niov+nvsz-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); - if (q->v_start != q->v_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->v_start); - q->v_start = niov; - q->v_end = niov + nvsz; - q->v_tail = q->v_end; - q->v_head = q->v_tail - q_sz; - - sys_memcpy(nbinv+nvsz-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - if (q->b_start != q->b_small) - erts_free(ERTS_ALC_T_IOQ, (void *) q->b_start); - q->b_start = nbinv; - q->b_end = nbinv + nvsz; - q->b_tail = q->b_end; - q->b_head = q->b_tail - q_sz; - } - } - else if (tail) { /* move to beginning to make room in tail */ - sys_memmove(q->v_start, q->v_head, q_sz*sizeof(SysIOVec)); - q->v_head = q->v_start; - q->v_tail = q->v_head + q_sz; - sys_memmove(q->b_start, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - q->b_head = q->b_start; - q->b_tail = q->b_head + q_sz; - } - else { /* move to end to make room */ - sys_memmove(q->v_end-q_sz, q->v_head, q_sz*sizeof(SysIOVec)); - q->v_tail = q->v_end; - q->v_head = q->v_tail-q_sz; - sys_memmove(q->b_end-q_sz, q->b_head, q_sz*sizeof(ErlDrvBinary*)); - q->b_tail = q->b_end; - q->b_head = q->b_tail-q_sz; - } - - return 0; -} - - - /* Put elements from vec at q tail */ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) { - int n; - size_t len; - ErlDrvSizeT size; - SysIOVec* iov; - ErlDrvBinary** binv; - ErlDrvBinary* b; - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) - return -1; - - ASSERT(vec->size >= skip); /* debug only */ - if (vec->size <= skip) - return 0; - size = vec->size - skip; - - iov = vec->iov; - binv = vec->binv; - n = vec->vsize; - - /* we use do here to strip iov_len=0 from beginning */ - do { - len = iov->iov_len; - if (len <= skip) { - skip -= len; - iov++; - binv++; - n--; - } - else { - iov->iov_base = ((char *)(iov->iov_base)) + skip; - iov->iov_len -= skip; - skip = 0; - } - } while(skip > 0); - - if (q->v_tail + n >= q->v_end) - expandq(q, n, 1); - - /* Queue and reference all binaries (remove zero length items) */ - while(n--) { - if ((len = iov->iov_len) > 0) { - if ((b = *binv) == NULL) { /* speical case create binary ! */ - b = driver_alloc_binary(len); - sys_memcpy(b->orig_bytes, iov->iov_base, len); - *q->b_tail++ = b; - q->v_tail->iov_len = len; - q->v_tail->iov_base = b->orig_bytes; - q->v_tail++; - } - else { - driver_binary_inc_refc(b); - *q->b_tail++ = b; - *q->v_tail++ = *iov; - } - } - iov++; - binv++; - } - q->size += size; /* update total size in queue */ - return 0; + ASSERT(vec->size >= skip); + return erts_ioq_enqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip); } /* Put elements from vec at q head */ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) { - int n; - size_t len; - ErlDrvSizeT size; - SysIOVec* iov; - ErlDrvBinary** binv; - ErlDrvBinary* b; - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) - return -1; - - if (vec->size <= skip) - return 0; - size = vec->size - skip; - - iov = vec->iov; - binv = vec->binv; - n = vec->vsize; - - /* we use do here to strip iov_len=0 from beginning */ - do { - len = iov->iov_len; - if (len <= skip) { - skip -= len; - iov++; - binv++; - n--; - } - else { - iov->iov_base = ((char *)(iov->iov_base)) + skip; - iov->iov_len -= skip; - skip = 0; - } - } while(skip > 0); - - if (q->v_head - n < q->v_start) - expandq(q, n, 0); - - /* Queue and reference all binaries (remove zero length items) */ - iov += (n-1); /* move to end */ - binv += (n-1); /* move to end */ - while(n--) { - if ((len = iov->iov_len) > 0) { - if ((b = *binv) == NULL) { /* speical case create binary ! */ - b = driver_alloc_binary(len); - sys_memcpy(b->orig_bytes, iov->iov_base, len); - *--q->b_head = b; - q->v_head--; - q->v_head->iov_len = len; - q->v_head->iov_base = b->orig_bytes; - } - else { - driver_binary_inc_refc(b); - *--q->b_head = b; - *--q->v_head = *iov; - } - } - iov--; - binv--; - } - q->size += size; /* update total size in queue */ - return 0; + ASSERT(vec->size >= skip); + return erts_ioq_pushqv(drvport2ioq(ix), (ErtsIOVec*)vec, skip); } - /* ** Remove size bytes from queue head ** Return number of bytes that remain in queue */ ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) { - ErlIOQueue* q = drvport2ioq(ix); - ErlDrvSizeT len; - - if ((q == NULL) || (q->size < size)) - return -1; - q->size -= size; - while (size > 0) { - ASSERT(q->v_head != q->v_tail); - - len = q->v_head->iov_len; - if (len <= size) { - size -= len; - driver_free_binary(*q->b_head); - *q->b_head++ = NULL; - q->v_head++; - } - else { - q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; - q->v_head->iov_len -= size; - size = 0; - } - } - - /* restart pointers (optimised for enq) */ - if (q->v_head == q->v_tail) { - q->v_head = q->v_tail = q->v_start; - q->b_head = q->b_tail = q->b_start; - } - return q->size; + ErlPortIOQueue *q = drvport2ioq(ix); + if (erts_ioq_deq(q, size) == -1) + return -1; + return erts_ioq_size(q); } -ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) { - ErlIOQueue *q = drvport2ioq(ix); - ASSERT(ev); - - if (! q) { - return (ErlDrvSizeT) -1; - } else { - if ((ev->vsize = q->v_tail - q->v_head) == 0) { - ev->size = 0; - ev->iov = NULL; - ev->binv = NULL; - } else { - ev->size = q->size; - ev->iov = q->v_head; - ev->binv = q->b_head; - } - return q->size; - } +ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) +{ + return erts_ioq_peekqv(drvport2ioq(ix), (ErtsIOVec*)ev); } SysIOVec* driver_peekq(ErlDrvPort ix, int* vlenp) /* length of io-vector */ { - ErlIOQueue* q = drvport2ioq(ix); - - if (q == NULL) { - *vlenp = -1; - return NULL; - } - if ((*vlenp = (q->v_tail - q->v_head)) == 0) - return NULL; - return q->v_head; + return erts_ioq_peekq(drvport2ioq(ix), vlenp); } ErlDrvSizeT driver_sizeq(ErlDrvPort ix) { - ErlIOQueue* q = drvport2ioq(ix); + ErlPortIOQueue *q = drvport2ioq(ix); if (q == NULL) - return (size_t) -1; - return q->size; + return (ErlDrvSizeT) -1; + return erts_ioq_size(q); } @@ -7537,7 +6853,7 @@ int driver_set_timer(ErlDrvPort ix, unsigned long t) { Port* prt = erts_drvport2port(ix); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; @@ -7554,7 +6870,7 @@ int driver_cancel_timer(ErlDrvPort ix) Port* prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); erts_cancel_port_timer(prt); return 0; } @@ -7565,11 +6881,11 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) Port* prt = erts_drvport2port(ix); Sint64 left; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); left = erts_read_port_timer(prt); if (left < 0) @@ -7584,7 +6900,7 @@ int driver_get_now(ErlDrvNowData *now_data) { Uint mega,secs,micro; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (now_data == NULL) { return -1; @@ -7658,7 +6974,7 @@ static int do_driver_monitor_process(Port *prt, erts_add_monitor(&ERTS_P_MONITORS(prt), MON_ORIGIN, ref, rp->common.id, NIL); erts_add_monitor(&ERTS_P_MONITORS(rp), MON_TARGET, ref, prt->common.id, NIL); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); erts_ref_to_driver_monitor(ref,monitor); return 0; } @@ -7672,7 +6988,7 @@ int driver_monitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -7682,7 +6998,7 @@ int driver_monitor_process(ErlDrvPort drvport, /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ - ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock)); ret = do_driver_monitor_process(prt,process,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; @@ -7717,7 +7033,7 @@ static int do_driver_demonitor_process(Port *prt, const ErlDrvMonitor *monitor) if (rp) { ErtsMonitor *rmon; rmon = erts_remove_monitor(&ERTS_P_MONITORS(rp), ref); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rmon != NULL) { erts_destroy_monitor(rmon); } @@ -7730,7 +7046,7 @@ int driver_demonitor_process(ErlDrvPort drvport, { Port *prt; int ret; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -7740,7 +7056,7 @@ int driver_demonitor_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ - ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock)); ret = do_driver_demonitor_process(prt,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; @@ -7771,7 +7087,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, { Port *prt; ErlDrvTermData ret; -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) ErtsSchedulerData *sched = erts_get_scheduler_data(); #endif @@ -7781,7 +7097,7 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort drvport, /* Now we should have either the port lock (if we have a scheduler) or the port data lock (if we're a driver thread) */ - ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock)); + ERTS_LC_ASSERT((sched != NULL || prt->port_data_lock)); ret = do_driver_get_monitored_process(prt,monitor); DRV_MONITOR_UNLOCK_PDL(prt); return ret; @@ -7802,7 +7118,7 @@ void erts_fire_port_monitor(Port *prt, Eterm ref) int fpe_was_unmasked; ERTS_MSACC_PUSH_STATE_M(); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); ASSERT(prt->drv_ptr != NULL); DRV_MONITOR_LOCK_PDL(prt); if (erts_lookup_monitor(ERTS_P_MONITORS(prt), ref) == NULL) { @@ -7849,11 +7165,11 @@ driver_failure_term(ErlDrvPort ix, Eterm term, int eof) erts_aint32_t state; Port* prt = erts_drvport2port_state(ix, &state); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if (prt->async_open_port) init_ack_send_reply(prt, prt->common.id); @@ -7888,7 +7204,7 @@ int driver_exit(ErlDrvPort ix, int err) ErtsLink *lnk, *rlnk = NULL; Eterm connected; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; @@ -7901,10 +7217,8 @@ int driver_exit(ErlDrvPort ix, int err) lnk = erts_remove_link(&ERTS_P_LINKS(prt), connected); -#ifdef ERTS_SMP if (rp) - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK); -#endif + erts_proc_unlock(rp, ERTS_PROC_LOCK_LINK); if (rlnk != NULL) { erts_destroy_link(rlnk); @@ -7958,7 +7272,7 @@ ErlDrvTermData driver_mk_atom(char* string) sys_strlen(string), ERTS_ATOM_ENC_LATIN1, 1); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; return (ErlDrvTermData) am; } @@ -7967,27 +7281,27 @@ ErlDrvTermData driver_mk_port(ErlDrvPort ix) Port* prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) return (ErlDrvTermData) NIL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); return (ErlDrvTermData) prt->common.id; } ErlDrvTermData driver_connected(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return NIL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); return ERTS_PORT_GET_CONNECTED(prt); } ErlDrvTermData driver_caller(ErlDrvPort ix) { Port* prt = erts_drvport2port(ix); - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return NIL; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); return prt->caller; } @@ -7996,20 +7310,20 @@ int driver_lock_driver(ErlDrvPort ix) Port* prt = erts_drvport2port(ix); DE_Handle* dh; - ERTS_SMP_CHK_NO_PROC_LOCKS; + ERTS_CHK_NO_PROC_LOCKS; if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - erts_smp_rwmtx_rwlock(&erts_driver_list_lock); + erts_rwmtx_rwlock(&erts_driver_list_lock); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) { - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); return -1; } erts_ddll_lock_driver(dh, prt->drv_ptr->name); - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); return 0; } @@ -8017,9 +7331,9 @@ int driver_lock_driver(ErlDrvPort ix) static int maybe_lock_driver_list(void) { void *rec_lock; - rec_lock = erts_smp_tsd_get(driver_list_lock_status_key); + rec_lock = erts_tsd_get(driver_list_lock_status_key); if (rec_lock == 0) { - erts_smp_rwmtx_rwlock(&erts_driver_list_lock); + erts_rwmtx_rwlock(&erts_driver_list_lock); return 1; } return 0; @@ -8027,7 +7341,7 @@ static int maybe_lock_driver_list(void) static void maybe_unlock_driver_list(int doit) { if (doit) { - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } } /* @@ -8050,7 +7364,7 @@ void *driver_dl_open(char * path) { void *ptr; int res; - int *last_error_p = erts_smp_tsd_get(driver_list_last_error_key); + int *last_error_p = erts_tsd_get(driver_list_last_error_key); int locked = maybe_lock_driver_list(); if ((res = erts_sys_ddll_open(path, &ptr, NULL)) == 0) { maybe_unlock_driver_list(locked); @@ -8058,7 +7372,7 @@ void *driver_dl_open(char * path) } else { if (!last_error_p) { last_error_p = erts_alloc(ERTS_ALC_T_DDLL_ERRCODES, sizeof(int)); - erts_smp_tsd_set(driver_list_last_error_key,last_error_p); + erts_tsd_set(driver_list_last_error_key,last_error_p); } *last_error_p = res; maybe_unlock_driver_list(locked); @@ -8070,7 +7384,7 @@ void *driver_dl_sym(void * handle, char *func_name) { void *ptr; int res; - int *last_error_p = erts_smp_tsd_get(driver_list_lock_status_key); + int *last_error_p = erts_tsd_get(driver_list_lock_status_key); int locked = maybe_lock_driver_list(); if ((res = erts_sys_ddll_sym(handle, func_name, &ptr)) == 0) { maybe_unlock_driver_list(locked); @@ -8078,7 +7392,7 @@ void *driver_dl_sym(void * handle, char *func_name) } else { if (!last_error_p) { last_error_p = erts_alloc(ERTS_ALC_T_DDLL_ERRCODES, sizeof(int)); - erts_smp_tsd_set(driver_list_lock_status_key,last_error_p); + erts_tsd_set(driver_list_lock_status_key,last_error_p); } *last_error_p = res; maybe_unlock_driver_list(locked); @@ -8098,7 +7412,7 @@ int driver_dl_close(void *handle) char *driver_dl_error(void) { char *res; - int *last_error_p = erts_smp_tsd_get(driver_list_lock_status_key); + int *last_error_p = erts_tsd_get(driver_list_lock_status_key); int locked = maybe_lock_driver_list(); res = erts_ddll_error((last_error_p != NULL) ? (*last_error_p) : ERL_DE_ERROR_UNSPECIFIED); maybe_unlock_driver_list(locked); @@ -8136,20 +7450,8 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) sip->driver_minor_version = ERL_DRV_EXTENDED_MINOR_VERSION; sip->erts_version = ERLANG_VERSION; sip->otp_release = ERLANG_OTP_RELEASE; - sip->thread_support = -#ifdef USE_THREADS - 1 -#else - 0 -#endif - ; - sip->smp_support = -#ifdef ERTS_SMP - 1 -#else - 0 -#endif - ; + sip->thread_support = 1; + sip->smp_support = 1; } @@ -8175,11 +7477,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) */ if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) { sip->dirty_scheduler_support = -#ifdef ERTS_DIRTY_SCHEDULERS 1 -#else - 0 -#endif ; } @@ -8206,14 +7504,6 @@ no_output_callback(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) } static void -no_event_callback(ErlDrvData drv_data, ErlDrvEvent event, ErlDrvEventData event_data) -{ - Port *prt = get_current_port(); - report_missing_drv_callback(prt, "Event", "event()"); - driver_event(ERTS_Port2ErlDrvPort(prt), event, NULL); -} - -static void no_ready_input_callback(ErlDrvData drv_data, ErlDrvEvent event) { Port *prt = get_current_port(); @@ -8259,25 +7549,17 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->version.minor = de->minor_version; drv->flags = de->driver_flags; drv->handle = handle; -#ifdef ERTS_SMP - if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) - drv->lock = NULL; - else { - drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, - sizeof(erts_mtx_t)); - erts_mtx_init_x(drv->lock, - "driver_lock", -#if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) - erts_atom_put((byte *) drv->name, - sys_strlen(drv->name), - ERTS_ATOM_ENC_LATIN1, - 1) -#else - NIL -#endif - ); + if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) { + drv->lock = NULL; + } else { + Eterm driver_id = erts_atom_put((byte *) drv->name, + sys_strlen(drv->name), + ERTS_ATOM_ENC_LATIN1, 1); + + drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, sizeof(erts_mtx_t)); + + erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO); } -#endif drv->entry = de; drv->start = de->start; @@ -8288,7 +7570,6 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) drv->outputv = de->outputv; drv->control = de->control; drv->call = de->call; - drv->event = de->event ? de->event : no_event_callback; drv->ready_input = de->ready_input ? de->ready_input : no_ready_input_callback; drv->ready_output = de->ready_output ? de->ready_output : no_ready_output_callback; drv->timeout = de->timeout ? de->timeout : no_timeout_callback; @@ -8320,12 +7601,10 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) void erts_destroy_driver(erts_driver_t *drv) { -#ifdef ERTS_SMP if (drv->lock) { - erts_smp_mtx_destroy(drv->lock); + erts_mtx_destroy(drv->lock); erts_free(ERTS_ALC_T_DRIVER_LOCK, drv->lock); } -#endif erts_free(ERTS_ALC_T_DRIVER, drv); } @@ -8336,7 +7615,7 @@ erts_destroy_driver(erts_driver_t *drv) void add_driver_entry(ErlDrvEntry *drv){ void *rec_lock; - rec_lock = erts_smp_tsd_get(driver_list_lock_status_key); + rec_lock = erts_tsd_get(driver_list_lock_status_key); /* * Ignore result of erts_add_driver_entry, the init is not * allowed to fail when drivers are added by drivers. @@ -8350,7 +7629,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo int res; if (!driver_list_locked) { - erts_smp_rwmtx_rwlock(&erts_driver_list_lock); + erts_rwmtx_rwlock(&erts_driver_list_lock); } dp->next = driver_list; @@ -8361,7 +7640,7 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo driver_list = dp; if (!driver_list_locked) { - erts_smp_tsd_set(driver_list_lock_status_key, (void *) 1); + erts_tsd_set(driver_list_lock_status_key, (void *) 1); } res = init_driver(dp, de, handle); @@ -8378,8 +7657,8 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle, int driver_list_lo } if (!driver_list_locked) { - erts_smp_tsd_set(driver_list_lock_status_key, NULL); - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_tsd_set(driver_list_lock_status_key, NULL); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } return res; } @@ -8390,9 +7669,9 @@ int remove_driver_entry(ErlDrvEntry *drv) erts_driver_t *dp; void *rec_lock; - rec_lock = erts_smp_tsd_get(driver_list_lock_status_key); + rec_lock = erts_tsd_get(driver_list_lock_status_key); if (rec_lock == NULL) { - erts_smp_rwmtx_rwlock(&erts_driver_list_lock); + erts_rwmtx_rwlock(&erts_driver_list_lock); } dp = driver_list; while (dp && dp->entry != drv) @@ -8400,7 +7679,7 @@ int remove_driver_entry(ErlDrvEntry *drv) if (dp) { if (dp->handle) { if (rec_lock == NULL) { - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } return -1; } @@ -8414,12 +7693,12 @@ int remove_driver_entry(ErlDrvEntry *drv) } erts_destroy_driver(dp); if (rec_lock == NULL) { - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } return 1; } if (rec_lock == NULL) { - erts_smp_rwmtx_rwunlock(&erts_driver_list_lock); + erts_rwmtx_rwunlock(&erts_driver_list_lock); } return 0; } diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab new file mode 100644 index 0000000000..0d175a7ec6 --- /dev/null +++ b/erts/emulator/beam/macros.tab @@ -0,0 +1,174 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +// +// Use if there is a garbage collection before storing to a +// general destination (either X or Y register). +// + +REFRESH_GEN_DEST() { + dst_ptr = REG_TARGET_PTR(dst); +} + +// $Offset is relative to the start of the instruction (not to the +// location of the failure label reference). Since combined +// instructions may increment the instruction pointer (e.g. in +// 'increment') for some of the instructions in the group, we actually +// use a virtual start position common to all instructions in the +// group. To calculate the correct virtual position, we will need to +// add $IP_ADJUSTMENT to the offset. ($IP_ADJUSTMENT will usually be +// zero, except in a few bit syntax instructions.) + +SET_I_REL(Offset) { + ASSERT(VALID_INSTR(*(I + ($Offset)))); + I += $Offset + $IP_ADJUSTMENT; +} + +SET_CP_I_ABS(Target) { + c_p->i = $Target; + ASSERT(VALID_INSTR(*c_p->i)); +} + +SET_REL_I(Dst, Offset) { + $Dst = I + ($Offset); + ASSERT(VALID_INSTR(*$Dst)); +} + +FAIL(Fail) { + //| -no_prefetch + $SET_I_REL($Fail); + Goto(*I); +} + +JUMP(Fail) { + //| -no_next + $SET_I_REL($Fail); + Goto(*I); +} + +GC_TEST(Ns, Nh, Live) { + Uint need = $Nh + $Ns; + if (ERTS_UNLIKELY(E - HTOP < need)) { + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + } + HEAP_SPACE_VERIFIED($Nh); +} + +GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) { + Uint need = $NeedHeap; + if (ERTS_UNLIKELY(E - HTOP < need)) { + SWAPOUT; + reg[$Live] = $PreserveTerm; + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live+1, FCALLS); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + $PreserveTerm = reg[$Live]; + SWAPIN; + } + HEAP_SPACE_VERIFIED($NeedHeap); +} + + +// Make sure that there are NeedStack + NeedHeap + 1 words available +// on the combined heap/stack segment, then allocates NeedHeap + 1 +// words on the stack and saves CP. +AH(NeedStack, NeedHeap, Live) { + unsigned needed = $NeedStack + 1; + $GC_TEST(needed, $NeedHeap, $Live); + E -= needed; + *E = make_cp(c_p->cp); + c_p->cp = 0; +} + +NEXT0() { + //| -no_next + SET_I((BeamInstr *) $NEXT_INSTRUCTION); + Goto(*I); +} + +NEXT(Addr) { + //| -no_next + SET_I((BeamInstr *) $Addr); + Goto(*I); +} + +FAIL_BODY() { + //| -no_prefetch + goto find_func_info; +} + +FAIL_HEAD_OR_BODY(Fail) { + //| -no_prefetch + + /* + * In a correctly working program, we expect failures in + * guards to be more likely than failures in bodies. + */ + + if (ERTS_LIKELY($Fail)) { + $FAIL($Fail); + } + goto find_func_info; +} + +BADARG(Fail) { + c_p->freason = BADARG; + $FAIL_HEAD_OR_BODY($Fail); +} + +BADARITH0() { + c_p->freason = BADARITH; + goto find_func_info; +} + +SYSTEM_LIMIT(Fail) { + c_p->freason = SYSTEM_LIMIT; + $FAIL_HEAD_OR_BODY($Fail); +} + +BIF_ERROR_ARITY_1(Fail, BIF, Op1) { + //| -no_prefetch + if (ERTS_LIKELY($Fail)) { + $FAIL($Fail); + } + reg[0] = $Op1; + SWAPOUT; + I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa); + goto post_error_handling; +} + +BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) { + //| -no_prefetch + if (ERTS_LIKELY($Fail)) { + $FAIL($Fail); + } + reg[0] = $Op1; + reg[1] = $Op2; + SWAPOUT; + I = handle_error(c_p, I, reg, &bif_export[$BIF]->info.mfa); + goto post_error_handling; +} diff --git a/erts/emulator/beam/map_instrs.tab b/erts/emulator/beam/map_instrs.tab new file mode 100644 index 0000000000..bbb2f49b66 --- /dev/null +++ b/erts/emulator/beam/map_instrs.tab @@ -0,0 +1,159 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +ensure_map(Map) { + if (is_not_map($Map)) { + c_p->freason = BADMAP; + c_p->fvalue = $Map; + $FAIL_BODY(); + } +} + +new_map(Dst, Live, N) { + Eterm res; + + HEAVY_SWAPOUT; + res = new_map(c_p, reg, $Live, $N, $NEXT_INSTRUCTION); + HEAVY_SWAPIN; + $REFRESH_GEN_DEST(); + $Dst = res; + $NEXT($NEXT_INSTRUCTION+$N); +} + +i_new_small_map_lit(Dst, Live, Keys) { + Eterm res; + Uint n; + Eterm keys = $Keys; + + HEAVY_SWAPOUT; + res = new_small_map_lit(c_p, reg, keys, $Live, $NEXT_INSTRUCTION); + HEAVY_SWAPIN; + $REFRESH_GEN_DEST(); + $Dst = res; + n = arityval(*tuple_val(keys)); + $NEXT($NEXT_INSTRUCTION+n); +} + +i_get_map_element(Fail, Src, Key, Dst) { + Eterm res = get_map_element($Src, $Key); + if (is_non_value(res)) { + $FAIL($Fail); + } + $Dst = res; +} + +i_get_map_element_hash(Fail, Src, Key, Hx, Dst) { + Eterm res = get_map_element_hash($Src, $Key, $Hx); + if (is_non_value(res)) { + $FAIL($Fail); + } + $Dst = res; +} + +i_get_map_elements(Fail, Src, N) { + Eterm map; + BeamInstr *fs; + Uint sz, n; + + map = $Src; + + /* This instruction assumes Arg1 is a map, + * i.e. that it follows a test is_map if needed. + */ + + n = (Uint)$N / 3; + fs = $NEXT_INSTRUCTION; + + if (is_flatmap(map)) { + flatmap_t *mp; + Eterm *ks; + Eterm *vs; + + mp = (flatmap_t *)flatmap_val(map); + sz = flatmap_get_size(mp); + + if (sz == 0) { + $FAIL($Fail); + } + + ks = flatmap_get_keys(mp); + vs = flatmap_get_values(mp); + + while(sz) { + if (EQ((Eterm) fs[0], *ks)) { + PUT_TERM_REG(*vs, fs[1]); + n--; + fs += 3; + /* no more values to fetch, we are done */ + if (n == 0) { + $NEXT(fs); + } + } + ks++, sz--, vs++; + } + $FAIL($Fail); + } else { + const Eterm *v; + Uint32 hx; + ASSERT(is_hashmap(map)); + while(n--) { + hx = fs[2]; + ASSERT(hx == hashmap_make_hash((Eterm)fs[0])); + if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) { + $FAIL($Fail); + } + PUT_TERM_REG(*v, fs[1]); + fs += 3; + } + $NEXT(fs); + } +} + +update_map_assoc(Src, Dst, Live, N) { + Eterm res; + Uint live = $Live; + + reg[live] = $Src; + HEAVY_SWAPOUT; + res = update_map_assoc(c_p, reg, live, $N, $NEXT_INSTRUCTION); + HEAVY_SWAPIN; + ASSERT(is_value(res)); + $REFRESH_GEN_DEST(); + $Dst = res; + $NEXT($NEXT_INSTRUCTION+$N); +} + +update_map_exact(Fail, Src, Dst, Live, N) { + Eterm res; + Uint live = $Live; + + reg[live] = $Src; + HEAVY_SWAPOUT; + res = update_map_exact(c_p, reg, live, $N, $NEXT_INSTRUCTION); + HEAVY_SWAPIN; + if (is_value(res)) { + $REFRESH_GEN_DEST(); + $Dst = res; + $NEXT($NEXT_INSTRUCTION+$N); + } else { + $FAIL_HEAD_OR_BODY($Fail); + } +} diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 8ab6c713d6..baeec115ea 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -39,9 +39,9 @@ static IndexTable module_tables[ERTS_NUM_CODE_IX]; -erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; +erts_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; -static erts_smp_atomic_t tot_module_bytes; +static erts_atomic_t tot_module_bytes; /* SMP note: Active module table lookup and current module instance can be * read without any locks. Old module instances are protected by @@ -49,8 +49,6 @@ static erts_smp_atomic_t tot_module_bytes; * Staging table is protected by the "code_ix lock". */ -#include "erl_smp.h" - void module_info(fmtfn_t to, void *to_arg) { index_info(to, to_arg, &module_tables[erts_active_code_ix()]); @@ -84,7 +82,7 @@ void erts_module_instance_init(struct erl_module_instance* modi) static Module* module_alloc(Module* tmpl) { Module* obj = (Module*) erts_alloc(ERTS_ALC_T_MODULE, sizeof(Module)); - erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); + erts_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; obj->slot.index = -1; @@ -98,7 +96,7 @@ static Module* module_alloc(Module* tmpl) static void module_free(Module* mod) { erts_free(ERTS_ALC_T_MODULE, mod); - erts_smp_atomic_add_nob(&tot_module_bytes, -sizeof(Module)); + erts_atomic_add_nob(&tot_module_bytes, -sizeof(Module)); } void init_module_table(void) @@ -120,9 +118,10 @@ void init_module_table(void) } for (i=0; i<ERTS_NUM_CODE_IX; i++) { - erts_smp_rwmtx_init_x(&the_old_code_rwlocks[i], "old_code", make_small(i)); + erts_rwmtx_init(&the_old_code_rwlocks[i], "old_code", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } - erts_smp_atomic_init_nob(&tot_module_bytes, 0); + erts_atomic_init_nob(&tot_module_bytes, 0); } @@ -158,14 +157,14 @@ static Module* put_module(Eterm mod, IndexTable* mod_tab) oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); newsz = index_table_sz(mod_tab); - erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); return res; } Module* erts_put_module(Eterm mod) { - ERTS_SMP_LC_ASSERT(erts_initialized == 0 + ERTS_LC_ASSERT(erts_initialized == 0 || erts_has_code_write_permission()); return put_module(mod, &module_tables[erts_staging_code_ix()]); @@ -183,7 +182,7 @@ int module_code_size(ErtsCodeIndex code_ix) int module_table_sz(void) { - return erts_smp_atomic_read_nob(&tot_module_bytes); + return erts_atomic_read_nob(&tot_module_bytes); } #ifdef DEBUG @@ -232,7 +231,7 @@ void module_start_staging(void) copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); - erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); entries_at_start_staging = dst->entries; IF_DEBUG(dbg_load_code_ix = erts_staging_code_ix()); @@ -250,7 +249,7 @@ void module_end_staging(int commit) oldsz = index_table_sz(tab); index_erase_latest_from(tab, entries_at_start_staging); newsz = index_table_sz(tab); - erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); + erts_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); } IF_DEBUG(dbg_load_code_ix = -1); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 9d258d5dbf..9a81e6035b 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -72,29 +72,29 @@ int erts_is_old_code_rlocked(ErtsCodeIndex); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; +extern erts_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX]; ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix) { - erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]); + erts_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]); } ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix) { - erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]); + erts_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]); } ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix) { - erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]); + erts_rwmtx_rlock(&the_old_code_rwlocks[code_ix]); } ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix) { - erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]); + erts_rwmtx_runlock(&the_old_code_rwlocks[code_ix]); } #ifdef ERTS_ENABLE_LOCK_CHECK ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix) { - return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]); + return erts_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]); } #endif diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab new file mode 100644 index 0000000000..8055a8616f --- /dev/null +++ b/erts/emulator/beam/msg_instrs.tab @@ -0,0 +1,390 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +// /* +// * Skeleton for receive statement: +// * +// * recv_mark L1 Optional +// * call make_ref/monitor Optional +// * ... +// * recv_set L1 Optional +// * L1: <-------------------+ +// * <-----------+ | +// * | | +// * loop_rec L2 ------+---+ | +// * ... | | | +// * remove_message | | | +// * jump L3 | | | +// * ... | | | +// * loop_rec_end L1 --+ | | +// * L2: <---------------+ | +// * wait L1 -------------------+ or wait_timeout +// * timeout +// * +// * L3: Code after receive... +// * +// */ + +recv_mark(Dest) { + /* + * Save the current position in message buffer and the + * the label for the loop_rec/2 instruction for the + * the receive statement. + */ + $SET_REL_I(c_p->msg.mark, $Dest); + c_p->msg.saved_last = c_p->msg.last; +} + +i_recv_set() { + /* + * If the mark is valid (points to the loop_rec/2 + * instruction that follows), we know that the saved + * position points to the first message that could + * possibly be matched out. + * + * If the mark is invalid, we do nothing, meaning that + * we will look through all messages in the message queue. + */ + if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) { + c_p->msg.save = c_p->msg.saved_last; + } + SET_I($NEXT_INSTRUCTION); + goto loop_rec_top__; + //| -no_next +} + +i_loop_rec(Dest) { + //| -no_prefetch + + /* + * Pick up the next message and place it in x(0). + * If no message, jump to a wait or wait_timeout instruction. + */ + + ErtsMessage* msgp; + + /* Entry point from recv_set */ + loop_rec_top__: + ; + + /* + * We need to disable GC while matching messages + * in the queue. This since messages with data outside + * the heap will be corrupted by a GC. + */ + ASSERT(!(c_p->flags & F_DELAY_GC)); + c_p->flags |= F_DELAY_GC; + + /* Entry point from loop_rec_end */ + loop_rec__: + + PROCESS_MAIN_CHK_LOCKS(c_p); + + msgp = PEEK_MESSAGE(c_p); + + if (!msgp) { + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + /* Make sure messages wont pass exit signals... */ + if (ERTS_PROC_PENDING_EXIT(c_p)) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + SWAPOUT; + c_p->flags &= ~F_DELAY_GC; + c_p->arity = 0; + goto do_schedule; /* Will be rescheduled for exit */ + } + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); + msgp = PEEK_MESSAGE(c_p); + if (msgp) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + } else { + c_p->flags &= ~F_DELAY_GC; + $SET_I_REL($Dest); + Goto(*I); /* Jump to a wait or wait_timeout instruction */ + } + } + if (is_non_value(ERL_MESSAGE_TERM(msgp))) { + SWAPOUT; /* erts_decode_dist_message() may write to heap... */ + if (!erts_decode_dist_message(c_p, ERTS_PROC_LOCK_MAIN, msgp, 0)) { + /* + * A corrupt distribution message that we weren't able to decode; + * remove it... + */ + /* No swapin should be needed */ + ASSERT(HTOP == c_p->htop && E == c_p->stop); + /* TODO: Add DTrace probe for this bad message situation? */ + UNLINK_MESSAGE(c_p, msgp); + msgp->next = NULL; + erts_cleanup_messages(msgp); + goto loop_rec__; + } + SWAPIN; + } + r(0) = ERL_MESSAGE_TERM(msgp); +} + +remove_message() { + //| -no_prefetch + + /* + * Remove a (matched) message from the message queue. + */ + + ErtsMessage* msgp; + PROCESS_MAIN_CHK_LOCKS(c_p); + + ERTS_CHK_MBUF_SZ(c_p); + + msgp = PEEK_MESSAGE(c_p); + + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + save_calls(c_p, &exp_receive); + } + if (ERL_MESSAGE_TOKEN(msgp) == NIL) { +#ifdef USE_VM_PROBES + if (DT_UTAG(c_p) != NIL) { + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) { + SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag; + } else { + DT_UTAG(c_p) = NIL; + SEQ_TRACE_TOKEN(c_p) = NIL; + } + } else { +#endif + SEQ_TRACE_TOKEN(c_p) = NIL; +#ifdef USE_VM_PROBES + } + DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING; +#endif + } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) { + Eterm msg; + SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp); +#ifdef USE_VM_PROBES + if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) { + if (DT_UTAG(c_p) == NIL) { + DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp); + } + DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING; + } else { +#endif + ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p))); + ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); + ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p))); + ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p))); + ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p))); + c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) { + c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p)); + } + msg = ERL_MESSAGE_TERM(msgp); + seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, + c_p->common.id, c_p); +#ifdef USE_VM_PROBES + } +#endif + } +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(message_receive)) { + Eterm token2 = NIL; + DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + + dtrace_proc_str(c_p, receiver_name); + token2 = SEQ_TRACE_TOKEN(c_p); + if (have_seqtrace(token2)) { + tok_label = signed_val(SEQ_TRACE_T_LABEL(token2)); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2)); + } + DTRACE6(message_receive, + receiver_name, size_object(ERL_MESSAGE_TERM(msgp)), + c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial); + } +#endif + UNLINK_MESSAGE(c_p, msgp); + JOIN_MESSAGE(c_p); + CANCEL_TIMER(c_p); + + erts_save_message_in_proc(c_p, msgp); + c_p->flags &= ~F_DELAY_GC; + + if (ERTS_IS_GC_DESIRED_INTERNAL(c_p, HTOP, E)) { + /* + * We want to GC soon but we leave a few + * reductions giving the message some time + * to turn into garbage. + */ + ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS); + } + + ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_CHK_MBUF_SZ(c_p); + + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); +} + +loop_rec_end(Dest) { + //| -no_next + /* + * Advance the save pointer to the next message (the current + * message didn't match), then jump to the loop_rec instruction. + */ + + ASSERT(c_p->flags & F_DELAY_GC); + + $SET_I_REL($Dest); + SAVE_MESSAGE(c_p); + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + goto loop_rec__; + } + + c_p->flags &= ~F_DELAY_GC; + $SET_CP_I_ABS(I); + SWAPOUT; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; +} + +timeout_locked() { + /* + * A timeout has occurred. Reset the save pointer so that the next + * receive statement will examine the first message first. + */ + + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + $timeout(); +} + +timeout() { + if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { + trace_receive(c_p, am_clock_service, am_timeout, NULL); + } + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) { + save_calls(c_p, &exp_timeout); + } + c_p->flags &= ~F_TIMO; + JOIN_MESSAGE(c_p); +} + +TIMEOUT_VALUE() { + c_p->freason = EXC_TIMEOUT_VALUE; + goto find_func_info; + //| -no_next +} + +i_wait_error_locked() { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + $TIMEOUT_VALUE(); +} + +i_wait_error() { + $TIMEOUT_VALUE(); +} + +wait_timeout_unlocked_int := wait.lock.int.execute; +wait_timeout_locked_int := wait.int.execute; + +wait_timeout_unlocked := wait.lock.src.execute; +wait_timeout_locked := wait.src.execute; + +wait_unlocked := wait.lock.execute; +wait_locked := wait.unlocked.execute; + +wait.lock() { + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); +} + +wait.unlocked() { +} + +wait.int(Int) { + /* + * If we have already set the timer, we must NOT set it again. Therefore, + * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. + */ + if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { + BeamInstr** pi = (BeamInstr **) c_p->def_arg_reg; + *pi = $NEXT_INSTRUCTION; + erts_set_proc_timer_uword(c_p, $Int); + } +} + +wait.src(Src) { + /* + * If we have already set the timer, we must NOT set it again. Therefore, + * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag. + */ + if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) { + Eterm timeout_value = $Src; + if (timeout_value == make_small(0)) { + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + $NEXT0(); + } else if (timeout_value == am_infinity) { + c_p->flags |= F_TIMO; + } else { + int tres = erts_set_proc_timer_term(c_p, timeout_value); + if (tres == 0) { + /* + * The timer routiner will set c_p->i to the value in + * c_p->def_arg_reg[0]. Note that it is safe to use this + * location because there are no living x registers in + * a receive statement. + */ + BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg; + *pi = $NEXT_INSTRUCTION; + } else { /* Wrong time */ + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->freason = EXC_TIMEOUT_VALUE; + goto find_func_info; + } + } + } +} + +// +// Prepare to wait indefinitely for a new message to arrive +// (or the time set above if falling through from above). +// When a new message arrives, control will be transferred +// the loop_rec instruction (at label L1). In case of +// of timeout, control will be transferred to the timeout +// instruction following the wait_timeout instruction. +// + +wait.execute(JumpTarget) { + $SET_REL_I(c_p->i, $JumpTarget); /* L1 */ + SWAPOUT; + c_p->arity = 0; + + if (!ERTS_PTMR_IS_TIMED_OUT(c_p)) { + erts_atomic32_read_band_relb(&c_p->state, + ~ERTS_PSFLG_ACTIVE); + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + c_p->current = NULL; + goto do_schedule; + //| -no_next +} diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index cdf9cb58b9..75ff40606b 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -59,6 +59,7 @@ put_tuple u==0 d => too_old_compiler # All the other instructions. # +%cold label L i_func_info I a a I int_code_end @@ -68,6 +69,8 @@ i_debug_breakpoint i_return_time_trace i_return_to_trace i_yield +trace_jump W +%hot return @@ -96,22 +99,19 @@ line Loc | func_info M F A => func_info M F A | line Loc line I - -%macro: allocate Allocate -pack -%macro: allocate_zero AllocateZero -pack -%macro: allocate_heap AllocateHeap -pack -%macro: allocate_heap_zero AllocateHeapZero -pack -%macro: test_heap TestHeap -pack - allocate t t allocate_heap t I t -deallocate I + +%cold +deallocate Q +%hot + init y allocate_zero t t allocate_heap_zero t I t trim N Remaining => i_trim N -i_trim I +i_trim t test_heap I t @@ -122,8 +122,6 @@ init2 y y init3 y y y init Y1 | init Y2 | init Y3 => init3 Y1 Y2 Y3 init Y1 | init Y2 => init2 Y1 Y2 -%macro: init2 Init2 -pack -%macro: init3 Init3 -pack # Selecting values @@ -160,28 +158,20 @@ is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val_bins x f I -i_select_val_bins y f I +i_select_val_bins xy f I -i_select_val_lins x f I -i_select_val_lins y f I +i_select_val_lins xy f I -i_select_val2 x f c c f f -i_select_val2 y f c c f f +i_select_val2 xy f c c -i_select_tuple_arity x f I -i_select_tuple_arity y f I +i_select_tuple_arity xy f I -i_select_tuple_arity2 x f A A f f -i_select_tuple_arity2 y f A A f f +i_select_tuple_arity2 xy f A A -i_jump_on_val_zero x f I -i_jump_on_val_zero y f I +i_jump_on_val_zero xy f I -i_jump_on_val x f I I -i_jump_on_val y f I I +i_jump_on_val xy f I W -%macro: get_list GetList -pack get_list xy xy xy # The following get_list instructions using x(0) are frequently used. @@ -201,31 +191,27 @@ try Y F => catch Y F try_case Y => try_end Y try_end y +%cold try_case_end s +%hot # Destructive set tuple element -set_tuple_element s d P +set_tuple_element s S P # Get tuple element -%macro: i_get_tuple_element GetTupleElement -pack i_get_tuple_element xy P x %cold i_get_tuple_element xy P y %hot -%macro: i_get_tuple_element2 GetTupleElement2 -pack i_get_tuple_element2 x P x - -%macro: i_get_tuple_element2y GetTupleElement2Y -pack i_get_tuple_element2y x P y y -%macro: i_get_tuple_element3 GetTupleElement3 -pack i_get_tuple_element3 x P x -%macro: is_number IsNumber -fail_action %cold is_number f x is_number f y @@ -236,6 +222,11 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f +# +# Expection rasing instructions. Infrequently executed. +# + +%cold case_end NotInX=cy => move NotInX x | case_end x badmatch NotInX=cy => move NotInX x | badmatch x @@ -257,9 +248,14 @@ i_raise badarg j system_limit j +%hot + +# +# Move instructions. +# + move C=cxy x==0 | jump Lbl => move_jump Lbl C -%macro: move_jump MoveJump -nonext move_jump f ncxy # Movement to and from the stack is common @@ -283,10 +279,6 @@ move_window X1=x X2=x X3=x X4=x Y1=y Y4=y | move X5=x Y5=y | succ(Y4,Y5) => \ move_window X1=x X2=x X3=x Y1=y Y3=y => move_window3 X1 X2 X3 Y1 move_window X1=x X2=x X3=x X4=x Y1=y Y4=y => move_window4 X1 X2 X3 X4 Y1 -%macro: move_window3 MoveWindow3 -pack -%macro: move_window4 MoveWindow4 -pack -%macro: move_window5 MoveWindow5 -pack - move_window3 x x x y move_window4 x x x x y move_window5 x x x x x y @@ -311,10 +303,8 @@ swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D -%macro: swap_temp SwapTemp -pack swap_temp x xy x -%macro: swap Swap -pack swap x xy move Src=x D1=x | move Src=x D2=x => move_dup Src D1 D2 @@ -358,17 +348,13 @@ move C=aiq X=x==2 => move_x2 C move_x1 c move_x2 c -%macro: move_shift MoveShift -pack move_shift x x x move_shift y x x move_shift x y x move_shift x x y -%macro: move_dup MoveDup -pack move_dup xy x xy -%macro: move2_par Move2Par -pack - move2_par x y x y move2_par y x y x move2_par x x x x @@ -380,7 +366,6 @@ move2_par y x x y move2_par x x y x move2_par y x x x -%macro: move3 Move3 -pack move3 x y x y x y move3 y x y x y x move3 x x x x x x @@ -390,7 +375,6 @@ move3 x x x x x x move S=n D=y => init D move S=c D=y => move S x | move x D -%macro:move Move -pack move x x move x y move y x @@ -410,13 +394,15 @@ move r y loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail -label L | wait_timeout Fail Src | smp_already_locked(L) => label L | i_wait_timeout_locked Fail Src -wait_timeout Fail Src => i_wait_timeout Fail Src -i_wait_timeout Fail Src=aiq => gen_literal_timeout(Fail, Src) -i_wait_timeout_locked Fail Src=aiq => gen_literal_timeout_locked(Fail, Src) +label L | wait_timeout Fail Src | smp_already_locked(L) => \ + label L | wait_timeout_locked Src Fail +wait_timeout Fail Src => wait_timeout_unlocked Src Fail + +wait_timeout_unlocked Src=aiq Fail => gen_literal_timeout(Fail, Src) +wait_timeout_locked Src=aiq Fail => gen_literal_timeout_locked(Fail, Src) label L | wait Fail | smp_already_locked(L) => label L | wait_locked Fail -wait Fail | smp() => wait_unlocked Fail +wait Fail => wait_unlocked Fail label L | timeout | smp_already_locked(L) => label L | timeout_locked @@ -425,15 +411,19 @@ timeout timeout_locked i_loop_rec f loop_rec_end f -wait f wait_locked f wait_unlocked f -i_wait_timeout f I -i_wait_timeout f s -i_wait_timeout_locked f I -i_wait_timeout_locked f s + +# Note that a timeout value must fit in 32 bits. +wait_timeout_unlocked_int I f +wait_timeout_unlocked s f +wait_timeout_locked_int I f +wait_timeout_locked s f + +%cold i_wait_error i_wait_error_locked +%hot send @@ -441,33 +431,35 @@ send # Optimized comparisons with one immediate/literal operand. # -is_eq_exact Lbl R=xy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl S S => +is_eq_exact Lbl C1=c C2=c => move C1 x | is_eq_exact Lbl x C2 +is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C + +is_eq_exact Lbl R=xy n => is_nil Lbl R +is_eq_exact Lbl R=xy C=ia => i_is_eq_exact_immed Lbl R C is_eq_exact Lbl R=xy C=q => i_is_eq_exact_literal Lbl R C +is_ne_exact Lbl S S => jump Lbl +is_ne_exact Lbl C1=c C2=c => move C1 x | is_ne_exact Lbl x C2 +is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C + is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C -%macro: i_is_eq_exact_immed EqualImmed -fail_action -i_is_eq_exact_immed f r c -i_is_eq_exact_immed f x c -i_is_eq_exact_immed f y c +i_is_eq_exact_immed f rxy c -i_is_eq_exact_literal f x c -i_is_eq_exact_literal f y c +i_is_eq_exact_literal f xy c -%macro: i_is_ne_exact_immed NotEqualImmed -fail_action -i_is_ne_exact_immed f x c -i_is_ne_exact_immed f y c +i_is_ne_exact_immed f xy c -i_is_ne_exact_literal f x c -i_is_ne_exact_literal f y c +i_is_ne_exact_literal f xy c is_eq_exact Lbl Y=y X=x => is_eq_exact Lbl X Y -%macro: is_eq_exact EqualExact -fail_action -pack is_eq_exact f x xy -is_eq_exact f s s +is_eq_exact f y y + +is_ne_exact f S S -%macro: is_lt IsLessThan -fail_action is_lt f x x is_lt f x c is_lt f c x @@ -475,7 +467,6 @@ is_lt f c x is_lt f s s %hot -%macro: is_ge IsGreaterEqual -fail_action is_ge f x x is_ge f x c is_ge f c x @@ -483,13 +474,8 @@ is_ge f c x is_ge f s s %hot -%macro: is_ne_exact NotEqualExact -fail_action -is_ne_exact f s s - -%macro: is_eq Equal -fail_action is_eq f s s -%macro: is_ne NotEqual -fail_action is_ne f s s # @@ -507,9 +493,7 @@ i_put_tuple Dst Arity Puts=* | put S => \ i_put_tuple/2 -%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple -i_put_tuple x I -i_put_tuple y I +i_put_tuple xy I # # The instruction "put_list Const [] Dst" were generated in rare @@ -518,8 +502,6 @@ i_put_tuple y I # put_list Const=c n Dst => move Const x | put_list x n Dst -%macro:put_list PutList -pack - put_list x n x put_list y n x put_list x x x @@ -560,6 +542,7 @@ put_list s s d # Some more only used by the emulator # +%cold normal_exit continue_exit apply_bif @@ -567,6 +550,7 @@ call_nif call_error_handler error_action_code return_trace +%hot # # Instruction transformations & folded instructions. @@ -577,27 +561,18 @@ return_trace move S x==0 | return => move_return S -%macro: move_return MoveReturn -nonext -move_return x -move_return c -move_return n +move_return xcn move S x==0 | deallocate D | return => move_deallocate_return S D -%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext -move_deallocate_return x Q -move_deallocate_return y Q -move_deallocate_return c Q -move_deallocate_return n Q +move_deallocate_return xycn Q deallocate D | return => deallocate_return D -%macro: deallocate_return DeallocateReturn -nonext deallocate_return Q test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y -%macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y # @@ -608,8 +583,6 @@ is_tagged_tuple Fail Literal=q Arity Atom => \ move Literal x | is_tagged_tuple Fail x Arity Atom is_tagged_tuple Fail=f c Arity Atom => jump Fail -%macro:is_tagged_tuple IsTaggedTuple -fail_action - is_tagged_tuple f rxy A a # Test tuple & arity (head) @@ -618,17 +591,13 @@ is_tuple Fail Literal=q => move Literal x | is_tuple Fail x is_tuple Fail=f c => jump Fail is_tuple Fail=f S=xy | test_arity Fail=f S=xy Arity => is_tuple_of_arity Fail S Arity -%macro:is_tuple_of_arity IsTupleOfArity -fail_action - is_tuple_of_arity f rxy A -%macro: is_tuple IsTuple -fail_action is_tuple f rxy test_arity Fail Literal=q Arity => move Literal x | test_arity Fail x Arity test_arity Fail=f c Arity => jump Fail -%macro: test_arity IsArity -fail_action test_arity f xy A get_tuple_element Reg=x P1 D1=x | get_tuple_element Reg=x P2 D2=x | \ @@ -650,16 +619,13 @@ is_integer Fail Literal=q => move Literal x | is_integer Fail x is_integer Fail=f S=x | allocate Need Regs => is_integer_allocate Fail S Need Regs -%macro: is_integer_allocate IsIntegerAllocate -fail_action -is_integer_allocate f x I I +is_integer_allocate f x t t -%macro: is_integer IsInteger -fail_action is_integer f xy is_list Fail=f n => is_list Fail Literal=q => move Literal x | is_list Fail x is_list Fail=f c => jump Fail -%macro: is_list IsList -fail_action is_list f x %cold is_list f y @@ -667,24 +633,16 @@ is_list f y is_nonempty_list Fail=f S=x | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs -%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack -is_nonempty_list_allocate f rx I t - -is_nonempty_list F=f x==0 | test_heap I1 I2 => is_non_empty_list_test_heap F I1 I2 - -%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack -is_non_empty_list_test_heap f I t +is_nonempty_list F=f x==0 | test_heap I1 I2 => is_nonempty_list_test_heap F I1 I2 is_nonempty_list Fail=f S=x | get_list S D1=x D2=x => \ is_nonempty_list_get_list Fail S D1 D2 -%macro: is_nonempty_list_get_list IsNonemptyListGetList -fail_action -pack +is_nonempty_list_allocate f rx t t +is_nonempty_list_test_heap f I t is_nonempty_list_get_list f rx x x - -%macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f xy -%macro: is_atom IsAtom -fail_action is_atom f x %cold is_atom f y @@ -692,7 +650,6 @@ is_atom f y is_atom Fail=f a => is_atom Fail=f niq => jump Fail -%macro: is_float IsFloat -fail_action is_float f x %cold is_float f y @@ -703,12 +660,10 @@ is_float Fail Literal=q => move Literal x | is_float Fail x is_nil Fail=f n => is_nil Fail=f qia => jump Fail -%macro: is_nil IsNil -fail_action is_nil f xy is_binary Fail Literal=q => move Literal x | is_binary Fail x is_binary Fail=f c => jump Fail -%macro: is_binary IsBinary -fail_action is_binary f x %cold is_binary f y @@ -719,28 +674,24 @@ is_bitstr Fail Term => is_bitstring Fail Term is_bitstring Fail Literal=q => move Literal x | is_bitstring Fail x is_bitstring Fail=f c => jump Fail -%macro: is_bitstring IsBitstring -fail_action is_bitstring f x %cold is_bitstring f y %hot is_reference Fail=f cq => jump Fail -%macro: is_reference IsRef -fail_action is_reference f x %cold is_reference f y %hot is_pid Fail=f cq => jump Fail -%macro: is_pid IsPid -fail_action is_pid f x %cold is_pid f y %hot is_port Fail=f cq => jump Fail -%macro: is_port IsPort -fail_action is_port f x %cold is_port f y @@ -751,22 +702,19 @@ is_boolean Fail=f a==am_false => is_boolean Fail=f ac => jump Fail %cold -%macro: is_boolean IsBoolean -fail_action is_boolean f xy %hot is_function2 Fail=f acq Arity => jump Fail is_function2 Fail=f Fun a => jump Fail -is_function2 f s s -%macro: is_function2 IsFunction2 -fail_action +is_function2 f S s # Allocating & initializing. allocate Need Regs | init Y => allocate_init Need Regs Y init Y1 | init Y2 => init2 Y1 Y2 -%macro: allocate_init AllocateInit -pack -allocate_init t I y +allocate_init t t y ################################################################# # External function and bif calls. @@ -1013,16 +961,18 @@ call_ext_last Ar Func D => i_call_ext_last Func D call_ext_only Ar Func => i_call_ext_only Func i_apply -i_apply_last P +i_apply_last Q i_apply_only i_apply_fun -i_apply_fun_last P +i_apply_fun_last Q i_apply_fun_only +%cold i_hibernate i_perf_counter +%hot call_bif e @@ -1045,19 +995,18 @@ bif2 Fail Bif S1 S2 Dst => i_bif2 Fail Bif S1 S2 Dst i_get_hash c I d i_get s d -%macro: self Self self xy -%macro: node Node node x %cold node y %hot -i_fast_element j x I d -i_fast_element j y I d +# Note: 'I' is sufficient because this instruction will only be used +# if the arity fits in 24 bits. +i_fast_element xy j I d -i_element j xy s d +i_element xy j s d bif1 f b s d bif1_body b s d @@ -1068,50 +1017,35 @@ i_bif2_body b s s d # Internal calls. # -move S=c x==0 | call Ar P=f => i_move_call S P -move S=s x==0 | call Ar P=f => move_call S P - -i_move_call c f +move S=cxy x==0 | call Ar P=f => move_call S P -%macro:move_call MoveCall -arg_f -size -nonext move_call/2 +move_call cxy f -move_call xy f - -move S=c x==0 | call_last Ar P=f D => i_move_call_last P D S move S x==0 | call_last Ar P=f D => move_call_last S P D -i_move_call_last f P c - -%macro:move_call_last MoveCallLast -arg_f -nonext -pack - move_call_last/3 -move_call_last xy f Q +move_call_last cxy f Q -move S=c x==0 | call_only Ar P=f => i_move_call_only P S -move S=x x==0 | call_only Ar P=f => move_call_only S P +move S=cx x==0 | call_only Ar P=f => move_call_only S P -i_move_call_only f c - -%macro:move_call_only MoveCallOnly -arg_f -nonext move_call_only/2 - -move_call_only x f +move_call_only cx f call Ar Func => i_call Func call_last Ar Func D => i_call_last Func D call_only Ar Func => i_call_only Func i_call f -i_call_last f P +i_call_last f Q i_call_only f i_call_ext e -i_call_ext_last e P +i_call_ext_last e Q i_call_ext_only e i_move_call_ext c e -i_move_call_ext_last e P c +i_move_call_ext_last e Q c i_move_call_ext_only e c # Fun calls. @@ -1119,17 +1053,15 @@ i_move_call_ext_only e c call_fun Arity | deallocate D | return => i_call_fun_last Arity D call_fun Arity => i_call_fun Arity -i_call_fun I -i_call_fun_last I P +i_call_fun t +i_call_fun_last t Q make_fun2 OldIndex=u => gen_make_fun2(OldIndex) -%macro: i_make_fun MakeFun -pack %cold -i_make_fun I t +i_make_fun W t %hot -%macro: is_function IsFunction -fail_action is_function f xy is_function Fail=f c => jump Fail @@ -1139,45 +1071,44 @@ func_info M F A => i_func_info u M F A # New bit syntax matching (R11B). # ================================================================ -%cold +%warm bs_start_match2 Fail=f ica X Y D => jump Fail bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D -i_bs_start_match2 xy f I I d +i_bs_start_match2 xy f t t x bs_save2 Reg Index => gen_bs_save(Reg, Index) -i_bs_save2 x I +i_bs_save2 x t bs_restore2 Reg Index => gen_bs_restore(Reg, Index) -i_bs_restore2 x I +i_bs_restore2 x t # Matching integers bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val -i_bs_match_string x f I I +i_bs_match_string x f W W # Fetching integers from binaries. bs_get_integer2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -i_bs_get_integer_small_imm x I f I d -i_bs_get_integer_imm x I I f I d -i_bs_get_integer f I I s s d -i_bs_get_integer_8 x f d -i_bs_get_integer_16 x f d -i_bs_get_integer_32 x f I d +i_bs_get_integer_small_imm x W f t x +i_bs_get_integer_imm x W t f t x +i_bs_get_integer f t t x s x +i_bs_get_integer_8 x f x +i_bs_get_integer_16 x f x + +%if ARCH_64 +i_bs_get_integer_32 x f x +%endif # Fetching binaries from binaries. bs_get_binary2 Fail=f Ms=x Live=u Sz=sq Unit=u Flags=u Dst=d => \ gen_get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) -%macro: i_bs_get_binary_imm2 BsGetBinaryImm_2 -fail_action -%macro: i_bs_get_binary2 BsGetBinary_2 -fail_action -%macro: i_bs_get_binary_all2 BsGetBinaryAll_2 -fail_action - -i_bs_get_binary_imm2 f x I I I d -i_bs_get_binary2 f x I s I d -i_bs_get_binary_all2 f x I I d -i_bs_get_binary_all_reuse x f I +i_bs_get_binary_imm2 f x t W t x +i_bs_get_binary2 f x t s t x +i_bs_get_binary_all2 f x t t x +i_bs_get_binary_all_reuse x f t # Fetching float from binaries. bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ @@ -1185,30 +1116,24 @@ bs_get_float2 Fail=f Ms=x Live=u Sz=s Unit=u Flags=u Dst=d => \ bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail -%macro: i_bs_get_float2 BsGetFloat2 -fail_action -i_bs_get_float2 f x I s I d +i_bs_get_float2 f x t s t x # Miscellanous bs_skip_bits2 Fail=f Ms=x Sz=sq Unit=u Flags=u => \ gen_skip_bits2(Fail, Ms, Sz, Unit, Flags) -%macro: i_bs_skip_bits_imm2 BsSkipBitsImm2 -fail_action -i_bs_skip_bits_imm2 f x I - -%macro: i_bs_skip_bits2 BsSkipBits2 -fail_action -i_bs_skip_bits2 f x xy I - -%macro: i_bs_skip_bits_all2 BsSkipBitsAll2 -fail_action -i_bs_skip_bits_all2 f x I +i_bs_skip_bits_imm2 f x W +i_bs_skip_bits2 f x xy t +i_bs_skip_bits_all2 f x t bs_test_tail2 Fail=f Ms=x Bits=u==0 => bs_test_zero_tail2 Fail Ms bs_test_tail2 Fail=f Ms=x Bits=u => bs_test_tail_imm2 Fail Ms Bits bs_test_zero_tail2 f x -bs_test_tail_imm2 f x I +bs_test_tail_imm2 f x W bs_test_unit F Ms Unit=u==8 => bs_test_unit8 F Ms -bs_test_unit f x I +bs_test_unit f x t bs_test_unit8 f x # An y register operand for bs_context_to_binary is rare, @@ -1222,14 +1147,14 @@ bs_context_to_binary x # Utf8/utf16/utf32 support. (R12B-5) # bs_get_utf8 Fail=f Ms=x u u Dst=d => i_bs_get_utf8 Ms Fail Dst -i_bs_get_utf8 x f d +i_bs_get_utf8 x f x bs_skip_utf8 Fail=f Ms=x u u => i_bs_get_utf8 Ms Fail x bs_get_utf16 Fail=f Ms=x u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst bs_skip_utf16 Fail=f Ms=x u Flags=u => i_bs_get_utf16 Ms Fail Flags x -i_bs_get_utf16 x f I d +i_bs_get_utf16 x f t x bs_get_utf32 Fail=f Ms=x Live=u Flags=u Dst=d => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ @@ -1238,22 +1163,18 @@ bs_skip_utf32 Fail=f Ms=x Live=u Flags=u => \ bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ i_bs_validate_unicode_retract Fail x Ms -i_bs_validate_unicode_retract j s s +i_bs_validate_unicode_retract j s S %hot # # Constructing binaries # -%cold +%warm bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail -bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst | should_gen_heap_bin(Sz) => \ - i_bs_init_heap_bin Sz Regs Dst bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst -bs_init2 Fail Sz=u Words Regs Flags Dst | should_gen_heap_bin(Sz) => \ - i_bs_init_heap_bin_heap Sz Words Regs Dst bs_init2 Fail Sz=u Words Regs Flags Dst => \ i_bs_init_heap Sz Words Regs Dst @@ -1262,15 +1183,13 @@ bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ bs_init2 Fail Sz Words Regs Flags Dst => \ i_bs_init_fail_heap Sz Words Fail Regs Dst -i_bs_init_fail xy j I d +i_bs_init_fail xy j t x -i_bs_init_fail_heap s I j I d +i_bs_init_fail_heap s I j t x -i_bs_init I I d -i_bs_init_heap_bin I I d +i_bs_init W t x -i_bs_init_heap I I I d -i_bs_init_heap_bin_heap I I I d +i_bs_init_heap W I t x bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail @@ -1283,16 +1202,16 @@ bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ bs_init_bits Fail Sz Words Regs Flags Dst => \ i_bs_init_bits_fail_heap Sz Words Fail Regs Dst -i_bs_init_bits_fail xy j I d +i_bs_init_bits_fail xy j t x -i_bs_init_bits_fail_heap s I j I d +i_bs_init_bits_fail_heap s I j t x -i_bs_init_bits I I d -i_bs_init_bits_heap I I I d +i_bs_init_bits W t x +i_bs_init_bits_heap W I t x bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D -bs_add j s s I d +bs_add j s s t x bs_append Fail Size Extra Live Unit Bin Flags Dst => \ move Bin x | i_bs_append Fail Extra Live Unit Size Dst @@ -1302,8 +1221,8 @@ bs_private_append Fail Size Unit Bin Flags Dst => \ bs_init_writable -i_bs_append j I I I s d -i_bs_private_append j I s s d +i_bs_append j I t t s x +i_bs_private_append j t s S x # # Storing integers into binaries. @@ -1312,11 +1231,8 @@ i_bs_private_append j I s s d bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ gen_put_integer(Fail, Sz, Unit, Flags, Src) -%macro: i_new_bs_put_integer NewBsPutInteger -%macro: i_new_bs_put_integer_imm NewBsPutIntegerImm - -i_new_bs_put_integer j s I s -i_new_bs_put_integer_imm j I I s +i_new_bs_put_integer j s t s +i_new_bs_put_integer_imm j W t s # # Utf8/utf16/utf32 support. (R12B-5) @@ -1324,17 +1240,17 @@ i_new_bs_put_integer_imm j I I s bs_utf8_size j Src=s Dst=d => i_bs_utf8_size Src Dst -i_bs_utf8_size s d +i_bs_utf8_size s x bs_utf16_size j Src=s Dst=d => i_bs_utf16_size Src Dst -i_bs_utf16_size s d +i_bs_utf16_size s x bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src i_bs_put_utf8 j s -bs_put_utf16 j I s +bs_put_utf16 j t s bs_put_utf32 Fail=j Flags=u Src=s => \ i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src @@ -1349,11 +1265,8 @@ bs_put_float Fail Sz=q Unit Flags Val => badarg Fail bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_float(Fail, Sz, Unit, Flags, Src) -%macro: i_new_bs_put_float NewBsPutFloat -%macro: i_new_bs_put_float_imm NewBsPutFloatImm - -i_new_bs_put_float j s I s -i_new_bs_put_float_imm j I I s +i_new_bs_put_float j s t s +i_new_bs_put_float_imm j W t s # # Storing binaries into binaries. @@ -1362,14 +1275,9 @@ i_new_bs_put_float_imm j I I s bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ gen_put_binary(Fail, Sz, Unit, Flags, Src) -%macro: i_new_bs_put_binary NewBsPutBinary -i_new_bs_put_binary j s I s - -%macro: i_new_bs_put_binary_imm NewBsPutBinaryImm -i_new_bs_put_binary_imm j I s - -%macro: i_new_bs_put_binary_all NewBsPutBinaryAll -i_new_bs_put_binary_all j s I +i_new_bs_put_binary j s t s +i_new_bs_put_binary_imm j W s +i_new_bs_put_binary_all j s t # # Warning: The i_bs_put_string and i_new_bs_put_string instructions @@ -1377,9 +1285,7 @@ i_new_bs_put_binary_all j s I # Don't change the instruction format unless you change the loader too. # -bs_put_string I I - -%hot +bs_put_string W W # # New floating point instructions (R8). @@ -1393,11 +1299,13 @@ fnegate p FR1 FR2 => i_fnegate FR1 FR2 fconv Arg=iqan Dst=l => move Arg x | fconv x Dst -fmove q l -fmove d l -fmove l d +fmove Arg=l Dst=d => fstore Arg Dst +fmove Arg=dq Dst=l => fload Arg Dst -fconv d l +fstore l d +fload Sq l + +fconv S l i_fadd l l l i_fsub l l l @@ -1407,50 +1315,87 @@ i_fnegate l l fclearerror | no_fpe_signals() => fcheckerror p | no_fpe_signals() => + +%unless NO_FPE_SIGNALS fcheckerror p => i_fcheckerror i_fcheckerror fclearerror +%endif + +%hot # # New apply instructions in R10B. # -apply I -apply_last I P +apply t +apply_last t Q # -# Map instructions in R17. +# Handle compatibility with OTP 17 here. # -sorted_put_map_assoc/5 -put_map_assoc F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ - sorted_put_map_assoc F Map Dst Live Size Rest +i_put_map_assoc/4 + +# We KNOW that in OTP 20 (actually OTP 18 and higher), a put_map_assoc instruction +# is always preceded by an is_map test. That means that put_map_assoc can never +# fail and does not need any failure label. + +put_map_assoc Fail Map Dst Live Size Rest=* | compiled_with_otp_20_or_higher() => \ + i_put_map_assoc Map Dst Live Size Rest + +# Translate the put_map_assoc instruction if the module was compiled by a compiler +# before 20. This is only necessary if the OTP 17 compiler was used, but we +# have no safe and relatively easy way to know whether OTP 18/19 was used. + +put_map_assoc Fail=p Map Dst Live Size Rest=* => \ + ensure_map Map | i_put_map_assoc Map Dst Live Size Rest +put_map_assoc Fail=f Map Dst Live Size Rest=* => \ + is_map Fail Map | i_put_map_assoc Map Dst Live Size Rest + +ensure_map Lit=q | literal_is_map(Lit) => +ensure_map Src=cqy => move Src x | ensure_map x + +%cold +ensure_map x +%hot + +# +# Map instructions. First introduced in R17. +# + +sorted_put_map_assoc/4 +i_put_map_assoc Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_assoc Map Dst Live Size Rest sorted_put_map_exact/5 put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ sorted_put_map_exact F Map Dst Live Size Rest -sorted_put_map_assoc j Map Dst Live Size Rest=* | is_empty_map(Map) => \ +sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \ new_map Dst Live Size Rest -sorted_put_map_assoc F Src=s Dst Live Size Rest=* => \ - update_map_assoc F Src Dst Live Size Rest -sorted_put_map_assoc F Src Dst Live Size Rest=* => \ - move Src x | update_map_assoc F x Dst Live Size Rest +sorted_put_map_assoc Src=s Dst Live Size Rest=* => \ + update_map_assoc Src Dst Live Size Rest +sorted_put_map_assoc Src Dst Live Size Rest=* => \ + move Src x | update_map_assoc x Dst Live Size Rest sorted_put_map_exact F Src=s Dst Live Size Rest=* => \ update_map_exact F Src Dst Live Size Rest sorted_put_map_exact F Src Dst Live Size Rest=* => \ move Src x | update_map_exact F x Dst Live Size Rest -new_map d I I -update_map_assoc j s d I I -update_map_exact j s d I I +new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \ + gen_new_small_map_lit(Dst, Live, Size, Rest) + +new_map d t I +i_new_small_map_lit d t q +update_map_assoc s d t I +update_map_exact j s d t I is_map Fail Lit=q | literal_is_map(Lit) => is_map Fail cq => jump Fail -%macro: is_map IsMap -fail_action is_map f xy ## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements @@ -1470,10 +1415,8 @@ i_get_map_elements f s I i_get_map_element Fail Src=xy Key=y Dst => \ move Key x | i_get_map_element Fail Src x Dst -%macro: i_get_map_element_hash GetMapElementHash -fail_action i_get_map_element_hash f xy c I xy -%macro: i_get_map_element GetMapElement -fail_action i_get_map_element f xy x xy # @@ -1509,9 +1452,9 @@ gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ # GCing arithmetic instructions. # -gen_plus Fail Live S1 S2 Dst => i_plus Fail Live S1 S2 Dst +gen_plus Fail Live S1 S2 Dst => i_plus S1 S2 Fail Live Dst -gen_minus Fail Live S1 S2 Dst => i_minus Fail Live S1 S2 Dst +gen_minus Fail Live S1 S2 Dst => i_minus S1 S2 Fail Live Dst gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \ i_times Fail Live S1 S2 Dst @@ -1522,15 +1465,15 @@ gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \ i_int_div Fail Live S1 S2 Dst gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \ - i_rem Fail Live S1 S2 Dst + i_rem S1 S2 Fail Live Dst gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \ - i_bsl Fail Live S1 S2 Dst + i_bsl S1 S2 Fail Live Dst gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \ - i_bsr Fail Live S1 S2 Dst + i_bsr S1 S2 Fail Live Dst gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \ - i_band Fail Live S1 S2 Dst + i_band S1 S2 Fail Live Dst gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \ i_bor Fail Live S1 S2 Dst @@ -1540,32 +1483,34 @@ gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst -i_increment rxy I I d +i_increment rxy W t d -i_plus j I x xy d -i_plus j I s s d +i_plus x xy j t d +i_plus s s j t d -i_minus j I x x d -i_minus j I s s d +i_minus x x j t d +i_minus s s j t d -i_times j I s s d +i_times j t s s d -i_m_div j I s s d -i_int_div j I s s d +i_m_div j t s s d +i_int_div j t s s d -i_rem j I x x d -i_rem j I s s d +i_rem x x j t d +i_rem s s j t d -i_bsl j I s s d -i_bsr j I s s d +i_bsl s s j t d +i_bsr s s j t d -i_band j I x c d -i_band j I s s d +i_band x c j t d +i_band s s j t d i_bor j I s s d i_bxor j I s s d -i_int_bnot j s I d +i_int_bnot Fail Src=c Live Dst => move Src x | i_int_bnot Fail x Live Dst + +i_int_bnot j S t d # # Old guard BIFs that creates heap fragments are no longer allowed. @@ -1589,9 +1534,9 @@ gc_bif2 Fail I Bif S1 S2 Dst => \ gc_bif3 Fail I Bif S1 S2 S3 Dst => \ gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) -i_gc_bif1 j I s I d +i_gc_bif1 j W s t d -i_gc_bif2 j I I s s d +i_gc_bif2 j W t s s d ii_gc_bif3/7 @@ -1600,7 +1545,7 @@ ii_gc_bif3/7 ii_gc_bif3 Fail Bif Live S1 S2 S3 Dst => \ move S1 x | i_gc_bif3 Fail Bif Live S2 S3 Dst -i_gc_bif3 j I I s s d +i_gc_bif3 j W t s s d # # The following instruction is specially handled in beam_load.c diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 7f60710124..92a0854ad3 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -38,16 +38,15 @@ static Hash process_reg; #define REG_HASH(term) ((HashValue) atom_val(term)) -static erts_smp_rwmtx_t regtab_rwmtx; +static erts_rwmtx_t regtab_rwmtx; -#define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) -#define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) -#define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) -#define reg_write_lock() erts_smp_rwmtx_rwlock(®tab_rwmtx) -#define reg_read_unlock() erts_smp_rwmtx_runlock(®tab_rwmtx) -#define reg_write_unlock() erts_smp_rwmtx_rwunlock(®tab_rwmtx) +#define reg_try_read_lock() erts_rwmtx_tryrlock(®tab_rwmtx) +#define reg_try_write_lock() erts_rwmtx_tryrwlock(®tab_rwmtx) +#define reg_read_lock() erts_rwmtx_rlock(®tab_rwmtx) +#define reg_write_lock() erts_rwmtx_rwlock(®tab_rwmtx) +#define reg_read_unlock() erts_rwmtx_runlock(®tab_rwmtx) +#define reg_write_unlock() erts_rwmtx_rwunlock(®tab_rwmtx) -#ifdef ERTS_SMP static ERTS_INLINE void reg_safe_read_lock(Process *c_p, ErtsProcLocks *c_p_locks) { @@ -64,7 +63,7 @@ reg_safe_read_lock(Process *c_p, ErtsProcLocks *c_p_locks) } /* Release process locks in order to avoid deadlock */ - erts_smp_proc_unlock(c_p, *c_p_locks); + erts_proc_unlock(c_p, *c_p_locks); *c_p_locks = 0; } @@ -87,14 +86,13 @@ reg_safe_write_lock(Process *c_p, ErtsProcLocks *c_p_locks) } /* Release process locks in order to avoid deadlock */ - erts_smp_proc_unlock(c_p, *c_p_locks); + erts_proc_unlock(c_p, *c_p_locks); *c_p_locks = 0; } reg_write_lock(); } -#endif static ERTS_INLINE int is_proc_alive(Process *p) @@ -141,11 +139,12 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; - rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; - rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + erts_rwmtx_opt_t rwmtx_opt = ERTS_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_RWMTX_LONG_LIVED; - erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); + erts_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; @@ -174,7 +173,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) Process *proc = NULL; Port *port = NULL; RegProc r, *rp; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); if (is_not_atom(name) || name == am_undefined) return res; @@ -184,7 +183,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) else { if (is_not_internal_pid(id) && is_not_internal_port(id)) return res; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); if (is_internal_port(id)) { port = erts_id2port(id); if (!port) @@ -192,15 +191,13 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) } } -#ifdef ERTS_SMP { ErtsProcLocks proc_locks = proc ? ERTS_PROC_LOCK_MAIN : 0; reg_safe_write_lock(proc, &proc_locks); if (proc && !proc_locks) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } -#endif if (is_internal_pid(id)) { if (!proc) @@ -214,7 +211,7 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) } else { ASSERT(!INVALID_PORT(port, id)); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(port)); r.pt = port; if (r.pt->common.u.alive.reg) goto done; @@ -249,8 +246,8 @@ int erts_register_name(Process *c_p, Eterm name, Eterm id) erts_port_release(port); if (c_p != proc) { if (proc) - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } return res; } @@ -271,17 +268,15 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) HashValue hval; int ix; HashBucket* b; -#ifdef ERTS_SMP ErtsProcLocks c_p_locks = 0; if (c_p) { c_p_locks = ERTS_PROC_LOCK_MAIN; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); } reg_safe_read_lock(c_p, &c_p_locks); if (c_p && !c_p_locks) - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); -#endif + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); hval = REG_HASH(name); ix = hval % process_reg.size; @@ -330,7 +325,6 @@ erts_whereis_name(Process *c_p, HashValue hval; int ix; HashBucket* b; -#ifdef ERTS_SMP ErtsProcLocks current_c_p_locks; Port *pending_port = NULL; @@ -347,7 +341,6 @@ erts_whereis_name(Process *c_p, * - read reg lock * - current_c_p_locks (either c_p_locks or 0) on c_p */ -#endif hval = REG_HASH(name); ix = hval % process_reg.size; @@ -369,7 +362,6 @@ erts_whereis_name(Process *c_p, if (!rp) *proc = NULL; else { -#ifdef ERTS_SMP if (!rp->p) *proc = NULL; else { @@ -386,17 +378,10 @@ erts_whereis_name(Process *c_p, *proc = rp->p; else { if (need_locks) - erts_smp_proc_unlock(rp->p, need_locks); + erts_proc_unlock(rp->p, need_locks); *proc = NULL; } } -#else - if (rp->p - && ((flags & ERTS_P2P_FLG_ALLOW_OTHER_X) || is_proc_alive(rp->p))) - *proc = rp->p; - else - *proc = NULL; -#endif if (*proc && (flags & ERTS_P2P_FLG_INC_REFC)) erts_proc_inc_refc(*proc); } @@ -406,7 +391,6 @@ erts_whereis_name(Process *c_p, if (!rp || !rp->pt) *port = NULL; else { -#ifdef ERTS_SMP if (lock_port) { if (pending_port == rp->pt) pending_port = NULL; @@ -418,11 +402,11 @@ erts_whereis_name(Process *c_p, pending_port = NULL; } - if (erts_smp_port_trylock(rp->pt) == EBUSY) { + if (erts_port_trylock(rp->pt) == EBUSY) { Eterm id = rp->pt->common.id; /* id read only... */ /* Unlock all locks, acquire port lock, and restart... */ if (current_c_p_locks) { - erts_smp_proc_unlock(c_p, current_c_p_locks); + erts_proc_unlock(c_p, current_c_p_locks); current_c_p_locks = 0; } reg_read_unlock(); @@ -430,19 +414,16 @@ erts_whereis_name(Process *c_p, goto restart; } } - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(rp->pt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(rp->pt)); } -#endif *port = rp->pt; } } -#ifdef ERTS_SMP if (c_p && !current_c_p_locks) - erts_smp_proc_lock(c_p, c_p_locks); + erts_proc_lock(c_p, c_p_locks); if (pending_port) erts_port_release(pending_port); -#endif reg_read_unlock(); } @@ -475,7 +456,6 @@ int erts_unregister_name(Process *c_p, RegProc r, *rp; Port *port = c_prt; ErtsProcLocks current_c_p_locks = 0; -#ifdef ERTS_SMP /* * SMP note: If 'c_prt != NULL' and 'c_prt->reg->name == name', @@ -491,18 +471,15 @@ int erts_unregister_name(Process *c_p, restart: reg_safe_write_lock(c_p, ¤t_c_p_locks); -#endif r.name = name; if (is_non_value(name)) { /* Unregister current process name */ ASSERT(c_p); -#ifdef ERTS_SMP if (current_c_p_locks != c_p_locks) { - erts_smp_proc_lock(c_p, c_p_locks); + erts_proc_lock(c_p, c_p_locks); current_c_p_locks = c_p_locks; } -#endif if (c_p->common.u.alive.reg) { r.name = c_p->common.u.alive.reg->name; } else { @@ -515,36 +492,34 @@ int erts_unregister_name(Process *c_p, if ((rp = (RegProc*) hash_get(&process_reg, (void*) &r)) != NULL) { if (rp->pt) { if (port != rp->pt) { -#ifdef ERTS_SMP if (port) { ASSERT(port != c_prt); erts_port_release(port); port = NULL; } - if (erts_smp_port_trylock(rp->pt) == EBUSY) { + if (erts_port_trylock(rp->pt) == EBUSY) { Eterm id = rp->pt->common.id; /* id read only... */ /* Unlock all locks, acquire port lock, and restart... */ if (current_c_p_locks) { - erts_smp_proc_unlock(c_p, current_c_p_locks); + erts_proc_unlock(c_p, current_c_p_locks); current_c_p_locks = 0; } reg_write_unlock(); port = erts_id2port(id); goto restart; } -#endif port = rp->pt; } ASSERT(rp->pt == port); - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(port)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(port)); rp->pt->common.u.alive.reg = NULL; if (IS_TRACED_FL(port, F_TRACE_PORTS)) { if (current_c_p_locks) { - erts_smp_proc_unlock(c_p, current_c_p_locks); + erts_proc_unlock(c_p, current_c_p_locks); current_c_p_locks = 0; } trace_port(port, am_unregister, r.name); @@ -552,7 +527,6 @@ int erts_unregister_name(Process *c_p, } else if (rp->p) { -#ifdef ERTS_SMP erts_proc_safelock(c_p, current_c_p_locks, c_p_locks, @@ -560,17 +534,14 @@ int erts_unregister_name(Process *c_p, (c_p == rp->p) ? current_c_p_locks : 0, ERTS_PROC_LOCK_MAIN); current_c_p_locks = c_p_locks; -#endif rp->p->common.u.alive.reg = NULL; if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { trace_proc(rp->p, (c_p == rp->p) ? c_p_locks : ERTS_PROC_LOCK_MAIN, rp->p, am_unregister, r.name); } -#ifdef ERTS_SMP if (rp->p != c_p) { - erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN); } -#endif } hash_erase(&process_reg, (void*) &r); res = 1; @@ -584,14 +555,12 @@ int erts_unregister_name(Process *c_p, erts_port_release(port); } if (c_prt) { - erts_smp_port_lock(c_prt); + erts_port_lock(c_prt); } } -#ifdef ERTS_SMP if (c_p && !current_c_p_locks) { - erts_smp_proc_lock(c_p, c_p_locks); + erts_proc_lock(c_p, c_p_locks); } -#endif return res; } @@ -632,14 +601,12 @@ BIF_RETTYPE registered_0(BIF_ALIST_0) Uint need; Eterm* hp; HashBucket **bucket; -#ifdef ERTS_SMP ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P); + ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P); reg_safe_read_lock(BIF_P, &proc_locks); if (!proc_locks) - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); -#endif + erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); bucket = process_reg.bucket; diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c index 30b26a7296..73306030ae 100644 --- a/erts/emulator/beam/safe_hash.c +++ b/erts/emulator/beam/safe_hash.c @@ -62,7 +62,7 @@ static ERTS_INLINE int align_up_pow2(int val) */ static void rehash(SafeHash* h, int grow_limit) { - if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) { + if (erts_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) { return; /* already in progress */ } if (h->grow_limit == grow_limit) { @@ -77,7 +77,7 @@ static void rehash(SafeHash* h, int grow_limit) sys_memzero(new_tab, bytes); for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { /* stop all traffic */ - erts_smp_mtx_lock(&h->lock_vec[i].mtx); + erts_mtx_lock(&h->lock_vec[i].mtx); } h->tab = new_tab; @@ -95,12 +95,12 @@ static void rehash(SafeHash* h, int grow_limit) } for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { - erts_smp_mtx_unlock(&h->lock_vec[i].mtx); + erts_mtx_unlock(&h->lock_vec[i].mtx); } erts_free(h->type, (void *) old_tab); } /*else already done */ - erts_smp_atomic_set_relb(&h->is_rehashing, 0); + erts_atomic_set_relb(&h->is_rehashing, 0); } @@ -115,7 +115,7 @@ void safe_hash_get_info(SafeHashInfo *hi, SafeHash *h) int objects = 0; for (lock_ix=0; lock_ix<SAFE_HASH_LOCK_CNT; lock_ix++) { - erts_smp_mtx_lock(&h->lock_vec[lock_ix].mtx); + erts_mtx_lock(&h->lock_vec[lock_ix].mtx); size = h->size_mask + 1; for (i = lock_ix; i < size; i += SAFE_HASH_LOCK_CNT) { int depth = 0; @@ -128,7 +128,7 @@ void safe_hash_get_info(SafeHashInfo *hi, SafeHash *h) if (depth > max_depth) max_depth = depth; } - erts_smp_mtx_unlock(&h->lock_vec[lock_ix].mtx); + erts_mtx_unlock(&h->lock_vec[lock_ix].mtx); } hi->name = h->name; @@ -145,9 +145,9 @@ int safe_hash_table_sz(SafeHash *h) int i, size; for(i=0; h->name[i]; i++); i++; - erts_smp_mtx_lock(&h->lock_vec[0].mtx); /* any lock will do to read size */ + erts_mtx_lock(&h->lock_vec[0].mtx); /* any lock will do to read size */ size = h->size_mask + 1; - erts_smp_mtx_unlock(&h->lock_vec[0].mtx); + erts_mtx_unlock(&h->lock_vec[0].mtx); return sizeof(SafeHash) + size*sizeof(SafeHashBucket*) + i; } @@ -155,7 +155,8 @@ int safe_hash_table_sz(SafeHash *h) ** Init a pre allocated or static hash structure ** and allocate buckets. NOT SAFE */ -SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size, SafeHashFunctions fun) +SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, erts_lock_flags_t flags, + int size, SafeHashFunctions fun) { int i, bytes; @@ -167,10 +168,11 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size, h->name = name; h->fun = fun; set_size(h,size); - erts_smp_atomic_init_nob(&h->is_rehashing, 0); - erts_smp_atomic_init_nob(&h->nitems, 0); + erts_atomic_init_nob(&h->is_rehashing, 0); + erts_atomic_init_nob(&h->nitems, 0); for (i=0; i<SAFE_HASH_LOCK_CNT; i++) { - erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash"); + erts_mtx_init(&h->lock_vec[i].mtx, "safe_hash", NIL, + flags); } return h; } @@ -183,8 +185,8 @@ void* safe_hash_get(SafeHash* h, void* tmpl) { SafeHashValue hval = h->fun.hash(tmpl); SafeHashBucket* b; - erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; - erts_smp_mtx_lock(lock); + erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; + erts_mtx_lock(lock); b = h->tab[hval & h->size_mask]; while(b != NULL) { @@ -192,7 +194,7 @@ void* safe_hash_get(SafeHash* h, void* tmpl) break; b = b->next; } - erts_smp_mtx_unlock(lock); + erts_mtx_unlock(lock); return (void*) b; } @@ -205,13 +207,13 @@ void* safe_hash_put(SafeHash* h, void* tmpl) SafeHashValue hval = h->fun.hash(tmpl); SafeHashBucket* b; SafeHashBucket** head; - erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; - erts_smp_mtx_lock(lock); + erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; + erts_mtx_lock(lock); head = &h->tab[hval & h->size_mask]; b = *head; while(b != NULL) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { - erts_smp_mtx_unlock(lock); + erts_mtx_unlock(lock); return b; } b = b->next; @@ -222,8 +224,8 @@ void* safe_hash_put(SafeHash* h, void* tmpl) b->next = *head; *head = b; grow_limit = h->grow_limit; - erts_smp_mtx_unlock(lock); - if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) { + erts_mtx_unlock(lock); + if (erts_atomic_inc_read_nob(&h->nitems) > grow_limit) { rehash(h, grow_limit); } return (void*) b; @@ -238,40 +240,58 @@ void* safe_hash_erase(SafeHash* h, void* tmpl) SafeHashValue hval = h->fun.hash(tmpl); SafeHashBucket* b; SafeHashBucket** prevp; - erts_smp_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; - erts_smp_mtx_lock(lock); + erts_mtx_t* lock = &h->lock_vec[hval % SAFE_HASH_LOCK_CNT].mtx; + erts_mtx_lock(lock); prevp = &h->tab[hval & h->size_mask]; b = *prevp; while(b != NULL) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { *prevp = b->next; - erts_smp_mtx_unlock(lock); - erts_smp_atomic_dec_nob(&h->nitems); + erts_mtx_unlock(lock); + erts_atomic_dec_nob(&h->nitems); h->fun.free((void*)b); return tmpl; } prevp = &b->next; b = b->next; } - erts_smp_mtx_unlock(lock); + erts_mtx_unlock(lock); return NULL; } /* -** Call 'func(obj,func_arg2)' for all objects in table. NOT SAFE!!! +** Call 'func(obj,func_arg2,func_arg3)' for all objects in table. NOT SAFE!!! */ -void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *), void *func_arg2) +void safe_hash_for_each(SafeHash* h, void (*func)(void *, void *, void *), + void *func_arg2, void *func_arg3) { int i; for (i = 0; i <= h->size_mask; i++) { SafeHashBucket* b = h->tab[i]; while (b != NULL) { - (*func)((void *) b, func_arg2); + (*func)((void *) b, func_arg2, func_arg3); b = b->next; } } } +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_enable_hash_lock_count(SafeHash *h, erts_lock_flags_t flags, int enable) { + int i; + + for(i = 0; i < SAFE_HASH_LOCK_CNT; i++) { + erts_mtx_t *lock = &h->lock_vec[i].mtx; + + if(enable) { + erts_lcnt_install_new_lock_info(&lock->lcnt, "safe_hash", NIL, + ERTS_LOCK_TYPE_MUTEX | flags); + } else { + erts_lcnt_uninstall(&lock->lcnt); + } + } +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ + #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index 285103cb17..af97b4cb4d 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -28,6 +28,7 @@ #include "sys.h" #include "erl_alloc.h" +#include "erl_lock_flags.h" typedef unsigned long SafeHashValue; @@ -72,11 +73,11 @@ typedef struct int size_mask; /* (RW) Number of slots - 1 */ SafeHashBucket** tab; /* (RW) Vector of bucket pointers (objects) */ int grow_limit; /* (RW) Threshold for growing table */ - erts_smp_atomic_t nitems; /* (A) Number of items in table */ - erts_smp_atomic_t is_rehashing; /* (A) Table rehashing in progress */ + erts_atomic_t nitems; /* (A) Number of items in table */ + erts_atomic_t is_rehashing; /* (A) Table rehashing in progress */ union { - erts_smp_mtx_t mtx; + erts_mtx_t mtx; byte __cache_line__[64]; }lock_vec[SAFE_HASH_LOCK_CNT]; @@ -85,7 +86,7 @@ typedef struct /* A: Lockless atomics */ } SafeHash; -SafeHash* safe_hash_init(ErtsAlcType_t, SafeHash*, char*, int, SafeHashFunctions); +SafeHash* safe_hash_init(ErtsAlcType_t, SafeHash*, char*, erts_lock_flags_t, int, SafeHashFunctions); void safe_hash_get_info(SafeHashInfo*, SafeHash*); int safe_hash_table_sz(SafeHash *); @@ -94,7 +95,11 @@ void* safe_hash_get(SafeHash*, void*); void* safe_hash_put(SafeHash*, void*); void* safe_hash_erase(SafeHash*, void*); -void safe_hash_for_each(SafeHash*, void (*func)(void *, void *), void *); +void safe_hash_for_each(SafeHash*, void (*func)(void *, void *, void *), void *, void *); + +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_enable_hash_lock_count(SafeHash*, erts_lock_flags_t, int); +#endif #endif /* __SAFE_HASH_H__ */ diff --git a/erts/emulator/beam/select_instrs.tab b/erts/emulator/beam/select_instrs.tab new file mode 100644 index 0000000000..2951949d38 --- /dev/null +++ b/erts/emulator/beam/select_instrs.tab @@ -0,0 +1,190 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +i_select_val_bins := select_val_bins.fetch.select; + +select_val_bins.head() { + Eterm select_val; +} + +select_val_bins.fetch(Src) { + select_val = $Src; +} + +select_val_bins.select(Fail, NumElements) { + struct Singleton { + BeamInstr val; + }; + struct Singleton* low; + struct Singleton* high; + struct Singleton* mid; + int bdiff; /* int not long because the arrays aren't that large */ + + low = (struct Singleton *) ($NEXT_INSTRUCTION); + high = low + $NumElements; + + /* The pointer subtraction (high-low) below must produce + * a signed result, because high could be < low. That + * requires the compiler to insert quite a bit of code. + * + * However, high will be > low so the result will be + * positive. We can use that knowledge to optimise the + * entire sequence, from the initial comparison to the + * computation of mid. + * + * -- Mikael Pettersson, Acumem AB + * + * Original loop control code: + * + * while (low < high) { + * mid = low + (high-low) / 2; + * + */ + while ((bdiff = (int)((char*)high - (char*)low)) > 0) { + unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Singleton)-1); + + mid = (struct Singleton*)((char*)low + boffset); + if (select_val < mid->val) { + high = mid; + } else if (select_val > mid->val) { + low = mid + 1; + } else { + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $NumElements); + Sint32 offset = jump_tab[mid - (struct Singleton *)($NEXT_INSTRUCTION)]; + $JUMP(offset); + } + } + $JUMP($Fail); +} + +i_select_tuple_arity2 := select_val2.src.get_arity.execute; +i_select_val2 := select_val2.src.execute; + +select_val2.head() { + Eterm select_val2; +} + +select_val2.src(Src) { + select_val2 = $Src; +} + +select_val2.get_arity() { + if (ERTS_LIKELY(is_tuple(select_val2))) { + select_val2 = *tuple_val(select_val2); + } else { + select_val2 = NIL; + } +} + +select_val2.execute(Fail, T1, T2) { + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION); + + if (select_val2 == $T1) { + $JUMP(jump_tab[0]); + } else if (select_val2 == $T2) { + $JUMP(jump_tab[1]); + } else { + $FAIL($Fail); + } +} + +i_select_tuple_arity := select_val_lin.fetch.get_arity.execute; +i_select_val_lins := select_val_lin.fetch.execute; + +select_val_lin.head() { + Eterm select_val; +} + +select_val_lin.fetch(Src) { + select_val = $Src; +} + +select_val_lin.get_arity() { + if (ERTS_LIKELY(is_tuple(select_val))) { + select_val = *tuple_val(select_val); + } else { + select_val = NIL; + } +} + +select_val_lin.execute(Fail, N) { + BeamInstr* vs = $NEXT_INSTRUCTION; + int ix = 0; + + for (;;) { + if (vs[ix+0] >= select_val) { + ix += 0; + break; + } + if (vs[ix+1] >= select_val) { + ix += 1; + break; + } + ix += 2; + } + + if (vs[ix] == select_val) { + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION + $N); + Eterm offset = jump_tab[ix]; + $JUMP(offset); + } else { + $JUMP($Fail); + } +} + +JUMP_ON_VAL(Fail, Index, N, Base) { + if (is_small($Index)) { + $Index = (Uint) (signed_val($Index) - $Base); + if ($Index < $N) { + Sint32* jump_tab = (Sint32 *) ($NEXT_INSTRUCTION); + $JUMP(jump_tab[$Index]); + } + } + $FAIL($Fail); +} + +i_jump_on_val_zero := jump_on_val_zero.fetch.execute; + +jump_on_val_zero.head() { + Eterm index; +} + +jump_on_val_zero.fetch(Src) { + index = $Src; +} + +jump_on_val_zero.execute(Fail, N) { + $JUMP_ON_VAL($Fail, index, $N, 0); +} + +i_jump_on_val := jump_on_val.fetch.execute; + +jump_on_val.head() { + Eterm index; +} + +jump_on_val.fetch(Src) { + index = $Src; +} + +jump_on_val.execute(Fail, N, Base) { + $JUMP_ON_VAL($Fail, index, $N, $Base); +} diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index d752ea4330..9106091ca6 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -21,7 +21,7 @@ #ifndef __SYS_H__ #define __SYS_H__ -#if !defined(__GNUC__) +#if !defined(__GNUC__) || defined(__e2k__) # define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0 #elif !defined(__GNUC_MINOR__) # define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \ @@ -34,9 +34,6 @@ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) #endif -#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ERTS_SMP) -# error "Dirty schedulers not supported without smp support" -#endif #ifdef ERTS_INLINE # ifndef ERTS_CAN_INLINE @@ -221,12 +218,6 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f # define ASSERT(e) ((void) 1) #endif -#ifdef ERTS_SMP -# define ERTS_SMP_ASSERT(e) ASSERT(e) -#else -# define ERTS_SMP_ASSERT(e) ((void)1) -#endif - /* ERTS_UNDEF can be used to silence false warnings about * "variable may be used uninitialized" while keeping the variable * marked as undefined by valgrind. @@ -327,9 +318,9 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #endif #if SIZEOF_VOID_P == SIZEOF_LONG -typedef unsigned long Eterm; -typedef unsigned long Uint; -typedef long Sint; +typedef unsigned long Eterm erts_align_attribute(sizeof(long)); +typedef unsigned long Uint erts_align_attribute(sizeof(long)); +typedef long Sint erts_align_attribute(sizeof(long)); #define SWORD_CONSTANT(Const) Const##L #define UWORD_CONSTANT(Const) Const##UL #define ERTS_UWORD_MAX ULONG_MAX @@ -337,9 +328,9 @@ typedef long Sint; #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT -typedef unsigned int Eterm; -typedef unsigned int Uint; -typedef int Sint; +typedef unsigned int Eterm erts_align_attribute(sizeof(int)); +typedef unsigned int Uint erts_align_attribute(sizeof(int)); +typedef int Sint erts_align_attribute(sizeof(int)); #define SWORD_CONSTANT(Const) Const #define UWORD_CONSTANT(Const) Const##U #define ERTS_UWORD_MAX UINT_MAX @@ -347,9 +338,9 @@ typedef int Sint; #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG -typedef unsigned long long Eterm; -typedef unsigned long long Uint; -typedef long long Sint; +typedef unsigned long long Eterm erts_align_attribute(sizeof(long long)); +typedef unsigned long long Uint erts_align_attribute(sizeof(long long)); +typedef long long Sint erts_align_attribute(sizeof(long long)); #define SWORD_CONSTANT(Const) Const##LL #define UWORD_CONSTANT(Const) Const##ULL #define ERTS_UWORD_MAX ULLONG_MAX @@ -470,41 +461,25 @@ typedef union { #include "erl_lock_check.h" -/* needed by erl_smp.h */ +/* needed by erl_threads.h */ int erts_send_warning_to_logger_str_nogl(char *); -#include "erl_smp.h" +#include "erl_threads.h" #ifdef ERTS_WANT_BREAK_HANDLING -# ifdef ERTS_SMP -extern erts_smp_atomic32_t erts_break_requested; +extern erts_atomic32_t erts_break_requested; # define ERTS_BREAK_REQUESTED \ - ((int) erts_smp_atomic32_read_nob(&erts_break_requested)) -# else -extern volatile int erts_break_requested; -# define ERTS_BREAK_REQUESTED erts_break_requested -# endif + ((int) erts_atomic32_read_nob(&erts_break_requested)) void erts_do_break_handling(void); #endif -#if !defined(ERTS_SMP) && !defined(__WIN32__) -extern volatile Uint erts_signal_state; -#define ERTS_SIGNAL_STATE erts_signal_state -void erts_handle_signal_state(void); -#endif -#ifdef ERTS_SMP -extern erts_smp_atomic32_t erts_writing_erl_crash_dump; +extern erts_atomic32_t erts_writing_erl_crash_dump; extern erts_tsd_key_t erts_is_crash_dumping_key; #define ERTS_SOMEONE_IS_CRASH_DUMPING \ - ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump)) + ((int) erts_atomic32_read_mb(&erts_writing_erl_crash_dump)) #define ERTS_IS_CRASH_DUMPING \ ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key)) -#else -extern volatile int erts_writing_erl_crash_dump; -#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump -#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump -#endif /* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions, but be able to fall back on bcopy() etc on systems that don't have @@ -600,8 +575,6 @@ __decl_noreturn void __noreturn erts_exit(int n, char*, ...); erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) -Eterm erts_check_io_info(void *p); - UWord erts_sys_get_page_size(void); /* Size of misc memory allocated from system dependent code */ @@ -643,7 +616,7 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_info_to_logger_str_nogl(char *); -/* needed by erl_smp.h (declared above) +/* needed by erl_threads.h (declared above) int erts_send_warning_to_logger_str_nogl(char *); */ int erts_send_error_to_logger_str_nogl(char *); @@ -764,11 +737,7 @@ extern char *erts_sys_ddll_error(int code); /* * System interfaces for startup. */ -void erts_sys_schedule_interrupt(int set); -#ifdef ERTS_SMP -void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime); void erts_sys_main_thread(void); -#endif extern int erts_sys_prepare_crash_dump(int secs); extern void erts_sys_pre_init(void); @@ -784,10 +753,6 @@ Preload* sys_preloaded(void); unsigned char* sys_preload_begin(Preload*); void sys_preload_end(Preload*); int sys_get_key(int); -void elapsed_time_both(UWord *ms_user, UWord *ms_sys, - UWord *ms_user_diff, UWord *ms_sys_diff); -void wall_clock_elapsed_time_both(UWord *ms_total, - UWord *ms_diff); void get_time(int *hour, int *minute, int *second); void get_date(int *year, int *month, int *day); void get_localtime(int *year, int *month, int *day, @@ -825,7 +790,7 @@ void fini_getenv_state(GETENV_STATE *); typedef struct { int no_used_fds; int no_driver_select_structs; - int no_driver_event_structs; + int no_enif_select_structs; } ErtsCheckIoDebugInfo; int erts_check_io_debug(ErtsCheckIoDebugInfo *ip); @@ -859,13 +824,11 @@ int erts_sys_unsetenv(char *key); char *erts_read_env(char *key); void erts_free_read_env(void *value); -#if defined(ERTS_SMP) #if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX) extern void sys_thr_resume(erts_tid_t tid); extern void sys_thr_suspend(erts_tid_t tid); #define ERTS_SYS_SUSPEND_SIGNAL SIGUSR2 #endif -#endif /* utils.c */ @@ -1029,144 +992,6 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -typedef erts_smp_atomic_t erts_smp_refc_t; - -ERTS_GLB_INLINE void erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val); -ERTS_GLB_INLINE void erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val); -ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inc_unless(erts_smp_refc_t *refcp, - erts_aint_t unless_val, - erts_aint_t min_val); -ERTS_GLB_INLINE erts_aint_t erts_smp_refc_inctest(erts_smp_refc_t *refcp, - erts_aint_t min_val); -ERTS_GLB_INLINE void erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val); -ERTS_GLB_INLINE erts_aint_t erts_smp_refc_dectest(erts_smp_refc_t *refcp, - erts_aint_t min_val); -ERTS_GLB_INLINE void erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, - erts_aint_t min_val); -ERTS_GLB_INLINE erts_aint_t erts_smp_refc_read(erts_smp_refc_t *refcp, - erts_aint_t min_val); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_smp_refc_init(erts_smp_refc_t *refcp, erts_aint_t val) -{ - erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val); -} - -ERTS_GLB_INLINE void -erts_smp_refc_inc(erts_smp_refc_t *refcp, erts_aint_t min_val) -{ -#ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#else - erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_refc_inc_unless(erts_smp_refc_t *refcp, - erts_aint_t unless_val, - erts_aint_t min_val) -{ - erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); - while (1) { - erts_aint_t exp, new; -#ifdef ERTS_REFC_DEBUG - if (val < 0) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_inc_unless(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#endif - if (val == unless_val) - return val; - new = val + 1; - exp = val; - val = erts_smp_atomic_cmpxchg_nob((erts_smp_atomic_t *) refcp, new, exp); - if (val == exp) - return new; - } -} - - -ERTS_GLB_INLINE erts_aint_t -erts_smp_refc_inctest(erts_smp_refc_t *refcp, erts_aint_t min_val) -{ - erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); -#ifdef ERTS_REFC_DEBUG - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#endif - return val; -} - -ERTS_GLB_INLINE void -erts_smp_refc_dec(erts_smp_refc_t *refcp, erts_aint_t min_val) -{ -#ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#else - erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_refc_dectest(erts_smp_refc_t *refcp, erts_aint_t min_val) -{ - erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); -#ifdef ERTS_REFC_DEBUG - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#endif - return val; -} - -ERTS_GLB_INLINE void -erts_smp_refc_add(erts_smp_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) -{ -#ifdef ERTS_REFC_DEBUG - erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", - diff, val, min_val); -#else - erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff); -#endif -} - -ERTS_GLB_INLINE erts_aint_t -erts_smp_refc_read(erts_smp_refc_t *refcp, erts_aint_t min_val) -{ - erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); -#ifdef ERTS_REFC_DEBUG - if (val < min_val) - erts_exit(ERTS_ABORT_EXIT, - "erts_smp_refc_read(): Bad refc found (refc=%ld < %ld)!\n", - val, min_val); -#endif - return val; -} - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - - -#ifdef ERTS_ENABLE_KERNEL_POLL -extern int erts_use_kernel_poll; -#endif - #define sys_memcpy(s1,s2,n) memcpy(s1,s2,n) #define sys_memmove(s1,s2,n) memmove(s1,s2,n) #define sys_memcmp(s1,s2,n) memcmp(s1,s2,n) @@ -1272,6 +1097,14 @@ void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) #endif +#ifndef MAX +#define MAX(A, B) ((A) > (B) ? (A) : (B)) +#endif + +#ifndef MIN +#define MIN(A, B) ((A) < (B) ? (A) : (B)) +#endif + #ifdef __WIN32__ #ifdef ARCH_64 #define ERTS_ALLOC_ALIGN_BYTES 16 diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab new file mode 100644 index 0000000000..3eee81c053 --- /dev/null +++ b/erts/emulator/beam/trace_instrs.tab @@ -0,0 +1,168 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + +return_trace() { + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); + + SWAPOUT; /* Needed for shared heap */ + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + SWAPIN; + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[2])); + E += 3; + Goto(*I); + //| -no_next +} + +i_generic_breakpoint() { + BeamInstr real_I; + HEAVY_SWAPOUT; + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); + HEAVY_SWAPIN; + ASSERT(VALID_INSTR(real_I)); + Goto(real_I); + //| -no_next +} + +i_return_time_trace() { + BeamInstr *pc = (BeamInstr *) (UWord) E[0]; + SWAPOUT; + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); + SWAPIN; + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[1])); + E += 2; + Goto(*I); + //| -no_next +} + +i_return_to_trace() { + if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { + Uint *cpp = (Uint*) E; + for(;;) { + ASSERT(is_CP(*cpp)); + if (IsOpCode(*cp_val(*cpp), return_trace)) { + do + ++cpp; + while (is_not_CP(*cpp)); + cpp += 2; + } else if (IsOpCode(*cp_val(*cpp), i_return_to_trace)) { + do + ++cpp; + while (is_not_CP(*cpp)); + } else { + break; + } + } + SWAPOUT; /* Needed for shared heap */ + ERTS_UNREQ_PROC_MAIN_LOCK(c_p); + erts_trace_return_to(c_p, cp_val(*cpp)); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + SWAPIN; + } + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[0])); + E += 1; + Goto(*I); + //| -no_next +} + +i_yield() { + /* This is safe as long as REDS_IN(c_p) is never stored + * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5]. + */ + c_p->arg_reg[0] = am_true; + c_p->arity = 1; /* One living register (the 'true' return value) */ + SWAPOUT; + $SET_CP_I_ABS($NEXT_INSTRUCTION); + c_p->current = NULL; + goto do_schedule; + //| -no_next +} + +i_hibernate() { + HEAVY_SWAPOUT; + if (erts_hibernate(c_p, reg)) { + FCALLS = c_p->fcalls; + c_p->flags &= ~F_HIBERNATE_SCHED; + goto do_schedule; + } else { + HEAVY_SWAPIN; + I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); + goto post_error_handling; + } + //| -no_next +} + +// This is optimised as an instruction because +// it has to be very very fast. + +i_perf_counter() { + ErtsSysPerfCounter ts; + + ts = erts_sys_perf_counter(); + if (IS_SSMALL(ts)) { + r(0) = make_small((Sint)ts); + } else { + $GC_TEST(0, ERTS_SINT64_HEAP_SIZE(ts), 0); + r(0) = make_big(HTOP); +#if defined(ARCH_32) + if (ts >= (((Uint64) 1) << 32)) { + *HTOP = make_pos_bignum_header(2); + BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); + BIG_DIGIT(HTOP, 1) = (Uint) ((ts >> 32) & ((Uint) 0xffffffff)); + HTOP += 3; + } + else +#endif + { + *HTOP = make_pos_bignum_header(1); + BIG_DIGIT(HTOP, 0) = (Uint) ts; + HTOP += 2; + } + } +} + +i_debug_breakpoint() { + HEAVY_SWAPOUT; + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); + HEAVY_SWAPIN; + if (I) { + Goto(*I); + } + goto handle_error; + //| -no_next +} + + + +// +// Special jump instruction used for tracing. Takes an absolute +// failure address. +// + +trace_jump(Fail) { + //| -no_next + SET_I((BeamInstr *) $Fail); + Goto(*I); +} diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 457cada745..993585be10 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -42,7 +42,7 @@ #include "dist.h" #include "erl_printf.h" #include "erl_threads.h" -#include "erl_smp.h" +#include "erl_lock_count.h" #include "erl_time.h" #include "erl_thr_progress.h" #include "erl_thr_queue.h" @@ -51,6 +51,7 @@ #include "erl_ptab.h" #include "erl_check_io.h" #include "erl_bif_unique.h" +#include "erl_io_queue.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #ifdef HIPE @@ -1930,27 +1931,6 @@ do_allocate_logger_message(Eterm gleader, Eterm **hp, ErlOffHeap **ohp, gl_sz = IS_CONST(gleader) ? 0 : size_object(gleader); sz = sz + gl_sz; -#ifndef ERTS_SMP -#ifdef USE_THREADS - if (!erts_get_scheduler_data()) /* Must be scheduler thread */ - *p = NULL; - else -#endif - { - *p = erts_whereis_process(NULL, 0, am_error_logger, 0, 0); - if (*p) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&(*p)->state); - if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) - *p = NULL; - } - } - - if (!*p) { - return NIL; - } - - /* So we have an error logger, lets build the message */ -#endif *bp = new_message_buffer(sz); *ohp = &(*bp)->off_heap; *hp = (*bp)->mem; @@ -1968,20 +1948,12 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment * #ifdef HARDDEBUG erts_fprintf(stderr, "%T\n", message); #endif -#ifdef ERTS_SMP { Eterm from = erts_get_current_pid(); if (is_not_internal_pid(from)) from = NIL; erts_queue_error_logger_message(from, message, bp); } -#else - { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = bp; - erts_queue_message(p, 0, mp, message, am_system); - } -#endif } /* error_logger ! @@ -3555,7 +3527,7 @@ store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) if (is_external_header(*from_hp)) { ExternalThing *etp = (ExternalThing *) from_hp; ASSERT(is_external(ns)); - erts_smp_refc_inc(&etp->node->refc, 2); + erts_refc_inc(&etp->node->refc, 2); } else if (is_ordinary_ref_thing(from_hp)) return make_internal_ref(to_hp); @@ -3634,13 +3606,78 @@ intlist_to_buf(Eterm list, char *buf, Sint len) return -2; /* not enough space */ } -/* Fill buf with the contents of the unicode list. - * Return the number of bytes in the buffer, - * or -1 for type error, - * or -2 for not enough buffer space (buffer contains truncated result). +/** @brief Fill buf with the UTF8 contents of the unicode list + * @param len Max number of characters to write. + * @param written NULL or bytes written. + * @return 0 ok, + * -1 type error, + * -2 list too long, only \c len characters written */ +int +erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len, Sint* written) +{ + Eterm* listptr; + Sint sz = 0; + Sint val; + int res; + + while (1) { + if (is_nil(list)) { + res = 0; + break; + } + if (is_not_list(list)) { + res = -1; + break; + } + listptr = list_val(list); + + if (len-- <= 0) { + res = -2; + break; + } + + if (is_not_small(CAR(listptr))) { + res = -1; + break; + } + val = signed_val(CAR(listptr)); + if (0 <= val && val < 0x80) { + buf[sz] = val; + sz++; + } else if (val < 0x800) { + buf[sz+0] = 0xC0 | (val >> 6); + buf[sz+1] = 0x80 | (val & 0x3F); + sz += 2; + } else if (val < 0x10000UL) { + if (0xD800 <= val && val <= 0xDFFF) { + res = -1; + break; + } + buf[sz+0] = 0xE0 | (val >> 12); + buf[sz+1] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+2] = 0x80 | (val & 0x3F); + sz += 3; + } else if (val < 0x110000) { + buf[sz+0] = 0xF0 | (val >> 18); + buf[sz+1] = 0x80 | ((val >> 12) & 0x3F); + buf[sz+2] = 0x80 | ((val >> 6) & 0x3F); + buf[sz+3] = 0x80 | (val & 0x3F); + sz += 4; + } else { + res = -1; + break; + } + list = CDR(listptr); + } + + if (written) + *written = sz; + return res; +} + Sint -erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) +erts_unicode_list_to_buf_len(Eterm list) { Eterm* listptr; Sint sz = 0; @@ -3653,7 +3690,7 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } listptr = list_val(list); - while (len-- > 0) { + while (1) { Sint val; if (is_not_small(CAR(listptr))) { @@ -3661,25 +3698,15 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } val = signed_val(CAR(listptr)); if (0 <= val && val < 0x80) { - buf[sz] = val; sz++; } else if (val < 0x800) { - buf[sz+0] = 0xC0 | (val >> 6); - buf[sz+1] = 0x80 | (val & 0x3F); sz += 2; } else if (val < 0x10000UL) { if (0xD800 <= val && val <= 0xDFFF) { return -1; } - buf[sz+0] = 0xE0 | (val >> 12); - buf[sz+1] = 0x80 | ((val >> 6) & 0x3F); - buf[sz+2] = 0x80 | (val & 0x3F); sz += 3; } else if (val < 0x110000) { - buf[sz+0] = 0xF0 | (val >> 18); - buf[sz+1] = 0x80 | ((val >> 12) & 0x3F); - buf[sz+2] = 0x80 | ((val >> 6) & 0x3F); - buf[sz+3] = 0x80 | (val & 0x3F); sz += 4; } else { return -1; @@ -3693,7 +3720,6 @@ erts_unicode_list_to_buf(Eterm list, byte *buf, Sint len) } listptr = list_val(list); } - return -2; /* not enough space */ } /* @@ -4656,22 +4682,6 @@ void erts_interval_init(erts_interval_t *icp) { erts_atomic64_init_nob(&icp->counter.atomic, 0); -#ifdef DEBUG - icp->smp_api = 0; -#endif -} - -void -erts_smp_interval_init(erts_interval_t *icp) -{ -#ifdef ERTS_SMP - erts_interval_init(icp); -#else - icp->counter.not_atomic = 0; -#endif -#ifdef DEBUG - icp->smp_api = 1; -#endif } static ERTS_INLINE Uint64 @@ -4711,81 +4721,27 @@ ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) Uint64 erts_step_interval_nob(erts_interval_t *icp) { - ASSERT(!icp->smp_api); return step_interval_nob(icp); } Uint64 erts_step_interval_relb(erts_interval_t *icp) { - ASSERT(!icp->smp_api); return step_interval_relb(icp); } Uint64 -erts_smp_step_interval_nob(erts_interval_t *icp) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return step_interval_nob(icp); -#else - return ++icp->counter.not_atomic; -#endif -} - -Uint64 -erts_smp_step_interval_relb(erts_interval_t *icp) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return step_interval_relb(icp); -#else - return ++icp->counter.not_atomic; -#endif -} - -Uint64 erts_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) { - ASSERT(!icp->smp_api); return ensure_later_interval_nob(icp, ic); } Uint64 erts_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) { - ASSERT(!icp->smp_api); return ensure_later_interval_acqb(icp, ic); } -Uint64 -erts_smp_ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return ensure_later_interval_nob(icp, ic); -#else - if (icp->counter.not_atomic > ic) - return icp->counter.not_atomic; - else - return ++icp->counter.not_atomic; -#endif -} - -Uint64 -erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) -{ - ASSERT(icp->smp_api); -#ifdef ERTS_SMP - return ensure_later_interval_acqb(icp, ic); -#else - if (icp->counter.not_atomic > ic) - return icp->counter.not_atomic; - else - return ++icp->counter.not_atomic; -#endif -} - /* * A millisecond timestamp without time correction where there's no hrtime * - for tracing on "long" things... diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 1538191d67..4e1d2f0d7f 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -167,7 +167,6 @@ dt_private *get_dt_private(int); #endif -#ifdef USE_THREADS #define THRDS_AVAILABLE (sys_info.async_threads > 0) #ifdef HARDDEBUG /* HARDDEBUG in io.c is expected too */ #define TRACE_DRIVER fprintf(stderr, "Efile: ") @@ -177,12 +176,6 @@ dt_private *get_dt_private(int); #define MUTEX_INIT(m, p) do { IF_THRDS { TRACE_DRIVER; (m = driver_pdl_create(p)); } } while (0) #define MUTEX_LOCK(m) do { IF_THRDS { TRACE_DRIVER; driver_pdl_lock(m); } } while (0) #define MUTEX_UNLOCK(m) do { IF_THRDS { TRACE_DRIVER; driver_pdl_unlock(m); } } while (0) -#else -#define THRDS_AVAILABLE (0) -#define MUTEX_INIT(m, p) -#define MUTEX_LOCK(m) -#define MUTEX_UNLOCK(m) -#endif #define IF_THRDS if (THRDS_AVAILABLE) @@ -742,7 +735,8 @@ file_init(void) efile_init(); #ifdef USE_VM_PROBES - erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex"); + erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); pthread_key_create(&dt_driver_key, NULL); #endif /* USE_VM_PROBES */ @@ -1267,6 +1261,50 @@ static void free_read_line(void *data) EF_FREE(d); } +void read_file_zero_size(struct t_data* d); +#define ZERO_FILE_CHUNK (64 * 1024) + +/* [ERL-327] Some special files like /proc/... have reported size 0 */ +void read_file_zero_size(struct t_data* d) { + size_t total_read_size = 0; + size_t allocated_size = ZERO_FILE_CHUNK; /* allocd in invoke_read_file */ + for (;;) { + size_t read_result; + + /* Read until we hit EOF (read less than FILE_SEGMENT_READ) */ + d->result_ok = efile_read(&d->errInfo, + EFILE_MODE_READ, + (int) d->fd, + (d->c.read_file.binp->orig_bytes + + total_read_size), + ZERO_FILE_CHUNK, + &read_result); + if (!d->result_ok) { + break; + } + + total_read_size += read_result; + d->c.read_file.offset += read_result; + if (read_result < ZERO_FILE_CHUNK) { + break; + } + + /* Grow before the next read call */ + allocated_size = total_read_size + ZERO_FILE_CHUNK; + d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, + allocated_size); + } + + /* Finalize the memory usage. Hopefully it was read fully on the first + * go, so the binary allocation overhead becomes: + * alloc ZERO_FILE_CHUNK (64kb) -> realloc real_size */ + if (allocated_size != total_read_size) { + d->c.read_file.binp = driver_realloc_binary(d->c.read_file.binp, + total_read_size); + } + d->again = 0; +} + static void invoke_read_file(void *data) { struct t_data *d = (struct t_data *) data; @@ -1285,9 +1323,15 @@ static void invoke_read_file(void *data) } d->fd = fd; d->c.read_file.size = (int) size; - if (size < 0 || size != d->c.read_file.size || - ! (d->c.read_file.binp = - driver_alloc_binary(d->c.read_file.size))) { + + /* For zero sized files allocate a reasonable chunk to attempt reading + * anyway. Note: This will eat ZERO_FILE_CHUNK bytes for any 0 file + * and free them immediately after (if the file was empty). */ + ERTS_ASSERT(size >= 0); + d->c.read_file.binp = driver_alloc_binary(size != 0 ? (size_t)size + : ZERO_FILE_CHUNK); + + if (size < 0 || size != d->c.read_file.size || !d->c.read_file.binp) { d->result_ok = 0; d->errInfo.posix_errno = ENOMEM; goto close; @@ -1296,6 +1340,11 @@ static void invoke_read_file(void *data) } /* Invariant: d->c.read_file.size >= d->c.read_file.offset */ + if (d->c.read_file.size == 0) { + read_file_zero_size(d); + goto close; + } + read_size = (size_t) (d->c.read_file.size - d->c.read_file.offset); if (! read_size) goto close; chop = d->again && read_size >= FILE_SEGMENT_READ*2; @@ -2714,7 +2763,6 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) } case FILE_READDIR: -#ifdef USE_THREADS if (sys_info.async_threads > 0) { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + @@ -2735,7 +2783,6 @@ file_output(ErlDrvData e, char* buf, ErlDrvSizeT count) goto done; } else -#endif { size_t resbufsize; size_t n = 0, total = 0; diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 13ee935e45..7b1f4a0e9c 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1249,6 +1249,8 @@ static int tcp_shutdown_error(tcp_descriptor* desc, int err); static int tcp_inet_output(tcp_descriptor* desc, HANDLE event); static int tcp_inet_input(tcp_descriptor* desc, HANDLE event); +static void tcp_desc_close(tcp_descriptor*); + #ifdef HAVE_UDP typedef struct { inet_descriptor inet; /* common data structure (DON'T MOVE) */ @@ -4334,6 +4336,12 @@ static void desc_close(inet_descriptor* desc) desc->event = INVALID_EVENT; /* closed by stop_select callback */ desc->s = INVALID_SOCKET; desc->event_mask = 0; + + /* mark as disconnected in case when socket is left lingering due to + * {exit_on_close, false} option in gen_tcp socket creation. Next + * write to socket should produce {error, enotconn} and send a + * message {tcp_error,#Port<>,econnreset} */ + desc->state &= ~INET_STATE_CONNECTED; } } @@ -9242,16 +9250,31 @@ static void tcp_inet_stop(ErlDrvData e) tcp_descriptor* desc = (tcp_descriptor*)e; DEBUGF(("tcp_inet_stop(%ld) {s=%d\r\n", (long)desc->inet.port, desc->inet.s)); + tcp_close_check(desc); - /* free input buffer & output buffer */ - if (desc->i_buf != NULL) - release_buffer(desc->i_buf); - desc->i_buf = NULL; /* net_mess2 may call this function recursively when - faulty messages arrive on dist ports*/ + tcp_clear_input(desc); + DEBUGF(("tcp_inet_stop(%ld) }\r\n", (long)desc->inet.port)); inet_stop(INETP(desc)); } +/* Closes a tcp descriptor without leaving things hanging; the VM keeps trying + * to flush IO queues as long as it contains anything even after the port has + * been closed from the erlang side, which is desired behavior (Think escripts + * writing to files) but pretty hopeless if the underlying fd has been set to + * INVALID_SOCKET through desc_close. + * + * This function should be used in place of desc_close/erl_inet_close in all + * TCP-related operations. Note that this only closes the desc cleanly; it + * will be freed through tcp_inet_stop later on. */ +static void tcp_desc_close(tcp_descriptor* desc) +{ + tcp_clear_input(desc); + tcp_clear_output(desc); + + erl_inet_close(INETP(desc)); +} + /* TCP requests from Erlang */ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, ErlDrvSizeT len, @@ -9496,7 +9519,7 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, case INET_REQ_CLOSE: DEBUGF(("tcp_inet_ctl(%ld): CLOSE\r\n", (long)desc->inet.port)); tcp_close_check(desc); - erl_inet_close(INETP(desc)); + tcp_desc_close(desc); return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); @@ -9620,7 +9643,7 @@ static void tcp_inet_timeout(ErlDrvData e) set_busy_port(desc->inet.port, 0); inet_reply_error_am(INETP(desc), am_timeout); if (desc->send_timeout_close) { - erl_inet_close(INETP(desc)); + tcp_desc_close(desc); } } else { @@ -9634,7 +9657,7 @@ static void tcp_inet_timeout(ErlDrvData e) else if ((state & INET_STATE_CONNECTING) == INET_STATE_CONNECTING) { /* assume connect timeout */ /* close the socket since it's not usable (see man pages) */ - erl_inet_close(INETP(desc)); + tcp_desc_close(desc); async_error_am(INETP(desc), am_timeout); } else if ((state & INET_STATE_ACCEPTING) == INET_STATE_ACCEPTING) { @@ -9797,8 +9820,7 @@ static int tcp_recv_closed(tcp_descriptor* desc) /* passive mode do not terminate port ! */ tcp_clear_input(desc); if (desc->inet.exitf) { - tcp_clear_output(desc); - desc_close(INETP(desc)); + tcp_desc_close(desc); } else { desc_close_read(INETP(desc)); } @@ -9841,7 +9863,7 @@ static int tcp_recv_error(tcp_descriptor* desc, int err) driver_cancel_timer(desc->inet.port); tcp_clear_input(desc); if (desc->inet.exitf) { - desc_close(INETP(desc)); + tcp_desc_close(desc); } else { desc_close_read(INETP(desc)); } @@ -10490,9 +10512,6 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) set_busy_port(desc->inet.port, 0); } - tcp_clear_output(desc); - tcp_clear_input(desc); - /* * We used to handle "expected errors" differently from unexpected ones. * Now we handle all errors in the same way (unless the show_econnreset @@ -10513,10 +10532,10 @@ static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) if (desc->inet.exitf) driver_exit(desc->inet.port, 0); else - desc_close(INETP(desc)); + tcp_desc_close(desc); } else { tcp_close_check(desc); - erl_inet_close(INETP(desc)); + tcp_desc_close(desc); if (desc->inet.caller) { if (show_econnreset) diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c deleted file mode 100644 index e342e414b5..0000000000 --- a/erts/emulator/drivers/common/zlib_drv.c +++ /dev/null @@ -1,792 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2017. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -/* - * ZLib interface for erlang - * - */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include <stdio.h> -#include <zlib.h> -#include <errno.h> -#include <string.h> - -#include "erl_driver.h" - - -#define DEFLATE_INIT 1 -#define DEFLATE_INIT2 2 -#define DEFLATE_SETDICT 3 -#define DEFLATE_RESET 4 -#define DEFLATE_END 5 -#define DEFLATE_PARAMS 6 -#define DEFLATE 7 - -#define INFLATE_INIT 8 -#define INFLATE_INIT2 9 -#define INFLATE_SETDICT 10 -#define INFLATE_GETDICT 11 -#define INFLATE_SYNC 12 -#define INFLATE_RESET 13 -#define INFLATE_END 14 -#define INFLATE 15 - -#define CRC32_0 16 -#define CRC32_1 17 -#define CRC32_2 18 - -#define SET_BUFSZ 19 -#define GET_BUFSZ 20 -#define GET_QSIZE 21 - -#define ADLER32_1 22 -#define ADLER32_2 23 - -#define CRC32_COMBINE 24 -#define ADLER32_COMBINE 25 - -#define INFLATE_CHUNK 26 - - -#define DEFAULT_BUFSZ 4000 - -/* According to zlib documentation, it can never exceed this */ -#define INFL_DICT_SZ 32768 - -/* This flag is used in the same places, where zlib return codes - * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to - * relatively large value to avoid possible value clashes in future. - * */ -#define INFLATE_HAS_MORE 100 - -static int zlib_init(void); -static ErlDrvData zlib_start(ErlDrvPort port, char* buf); -static void zlib_stop(ErlDrvData e); -static void zlib_flush(ErlDrvData e); -static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, - ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen); -static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev); - -ErlDrvEntry zlib_driver_entry = { - zlib_init, - zlib_start, - zlib_stop, - NULL, /* output */ - NULL, /* ready_input */ - NULL, /* ready_output */ - "zlib_drv", - NULL, /* finish */ - NULL, /* handle */ - zlib_ctl, - NULL, /* timeout */ - zlib_outputv, - NULL, /* read_async */ - zlib_flush, - NULL, /* call */ - NULL, /* event */ - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - ERL_DRV_FLAG_USE_PORT_LOCKING, - NULL, /* handle2 */ - NULL, /* process_exit */ -}; - -typedef enum { - ST_NONE = 0, - ST_DEFLATE = 1, - ST_INFLATE = 2 -} ZLibState; - - -typedef struct { - z_stream s; - ZLibState state; - ErlDrvBinary* bin; - int binsz; - int binsz_need; - uLong crc; - int inflate_eos_seen; - int want_crc; /* 1 if crc is calculated on clear text */ - ErlDrvPort port; /* the associcated port */ -} ZLibData; - -static int zlib_inflate(ZLibData* d, int flush); -static int zlib_deflate(ZLibData* d, int flush); - -#if defined(__WIN32__) -static int i32(char* buf) -#else -static __inline__ int i32(char* buf) -#endif -{ - return (int) ( - (((int)((unsigned char*)buf)[0]) << 24) | - (((int)((unsigned char*)buf)[1]) << 16) | - (((int)((unsigned char*)buf)[2]) << 8) | - (((int)((unsigned char*)buf)[3]) << 0)); -} - -static char* zlib_reason(int code, int* err) -{ - switch(code) { - case Z_OK: - *err = 0; - return "ok"; - case Z_STREAM_END: - *err = 0; - return "stream_end"; - case Z_ERRNO: - *err = 1; - return erl_errno_id(errno); - case Z_STREAM_ERROR: - *err = 1; - return "stream_error"; - case Z_DATA_ERROR: - *err = 1; - return "data_error"; - case Z_MEM_ERROR: - *err = 1; - return "mem_error"; - case Z_BUF_ERROR: - *err = 1; - return "buf_error"; - case Z_VERSION_ERROR: - *err = 1; - return "version_error"; - default: - *err = 1; - return "unknown_error"; - } -} - - -static ErlDrvSSizeT zlib_return(int code, char** rbuf, ErlDrvSizeT rlen) -{ - int msg_code = 0; /* 0=ok, 1=error */ - char* dst = *rbuf; - char* src; - ErlDrvSizeT len = 0; - - src = zlib_reason(code, &msg_code); - *dst++ = msg_code; - rlen--; - len = 1; - - while((rlen > 0) && *src) { - *dst++ = *src++; - rlen--; - len++; - } - return len; -} - -static ErlDrvSSizeT zlib_value2(int msg_code, int value, - char** rbuf, ErlDrvSizeT rlen) -{ - char* dst = *rbuf; - - if (rlen < 5) { - return -1; - } - *dst++ = msg_code; - *dst++ = (value >> 24) & 0xff; - *dst++ = (value >> 16) & 0xff; - *dst++ = (value >> 8) & 0xff; - *dst++ = value & 0xff; - return 5; -} - -static ErlDrvSSizeT zlib_value(int value, char** rbuf, ErlDrvSizeT rlen) -{ - return zlib_value2(2, value, rbuf, rlen); -} - -static int zlib_output_init(ZLibData* d) -{ - if (d->bin != NULL) - driver_free_binary(d->bin); - if ((d->bin = driver_alloc_binary(d->binsz_need)) == NULL) - return -1; - d->binsz = d->binsz_need; - d->s.next_out = (unsigned char*)d->bin->orig_bytes; - d->s.avail_out = d->binsz; - return 0; -} - -/* - * Send compressed or uncompressed data - * and restart output procesing - */ -static int zlib_output(ZLibData* d) -{ - if (d->bin != NULL) { - int len = d->binsz - d->s.avail_out; - if (len > 0) { - if (driver_output_binary(d->port, NULL, 0, d->bin, 0, len) < 0) - return -1; - } - driver_free_binary(d->bin); - d->bin = NULL; - d->binsz = 0; - } - return zlib_output_init(d); -} - -static int zlib_inflate_get_dictionary(ZLibData* d) -{ -#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY - ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ); - uInt dlen = 0; - int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen); - if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) { - res = Z_ERRNO; - } - driver_free_binary(dbin); - return res; -#else - abort(); /* never called, just to silence 'unresolved symbol' - for non-optimizing compiler */ -#endif -} - -static int zlib_inflate(ZLibData* d, int flush) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - int possibly_more_output = 0; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - while((possibly_more_output || (d->s.avail_in > 0)) && (res != Z_STREAM_END)) { - res = inflate(&d->s, Z_NO_FLUSH); - if (res == Z_NEED_DICT) { - /* Essential to eat the header bytes that zlib has looked at */ - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - return res; - } - if (res == Z_BUF_ERROR) { - /* Was possible more output, but actually not */ - res = Z_OK; - } - else if (res < 0) { - return res; - } - if (d->s.avail_out != 0) { - possibly_more_output = 0; - } else { - if (d->want_crc) - d->crc = crc32(d->crc, (unsigned char*)d->bin->orig_bytes, - d->binsz - d->s.avail_out); - zlib_output(d); - possibly_more_output = 1; - } - } - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - } - - if (d->want_crc) { - d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes, - d->binsz - d->s.avail_out); - } - zlib_output(d); - if (res == Z_STREAM_END) { - d->inflate_eos_seen = 1; - } - return res; -} - -static int zlib_inflate_chunk(ZLibData* d) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (d->s.avail_out > 0) && - (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - while((d->s.avail_in > 0) && (d->s.avail_out > 0) && (res != Z_STREAM_END)) { - res = inflate(&d->s, Z_NO_FLUSH); - if (res == Z_NEED_DICT) { - /* Essential to eat the header bytes that zlib has looked at */ - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - return res; - } - if (res == Z_BUF_ERROR) { - /* Was possible more output, but actually not */ - res = Z_OK; - } - else if (res < 0) { - return res; - } - } - len = iov[0].iov_len - d->s.avail_in; - driver_deq(d->port, len); - } - - /* We are here because all input was consumed or EOS reached or output - * buffer is full */ - if (d->want_crc) { - d->crc = crc32(d->crc, (unsigned char*) d->bin->orig_bytes, - d->binsz - d->s.avail_out); - } - zlib_output(d); - if ((res == Z_OK) && (d->s.avail_in > 0)) - res = INFLATE_HAS_MORE; - else if (res == Z_STREAM_END) { - d->inflate_eos_seen = 1; - } - return res; -} - -static int zlib_deflate(ZLibData* d, int flush) -{ - int res = Z_OK; - - if ((d->bin == NULL) && (zlib_output_init(d) < 0)) { - errno = ENOMEM; - return Z_ERRNO; - } - - while ((driver_sizeq(d->port) > 0) && (res != Z_STREAM_END)) { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - int len; - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - - while((d->s.avail_in > 0) && (res != Z_STREAM_END)) { - if ((res = deflate(&d->s, Z_NO_FLUSH)) < 0) { - return res; - } - if (d->s.avail_out == 0) { - zlib_output(d); - } - } - len = iov[0].iov_len - d->s.avail_in; - if (d->want_crc) { - d->crc = crc32(d->crc, iov[0].iov_base, len); - } - driver_deq(d->port, len); - } - - if (flush != Z_NO_FLUSH) { - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - if (flush == Z_FINISH) { - while (d->s.avail_out < d->binsz) { - zlib_output(d); - if (res == Z_STREAM_END) { - break; - } - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - } - } else { - while (d->s.avail_out == 0) { - zlib_output(d); - if ((res = deflate(&d->s, flush)) < 0) { - return res; - } - } - if (d->s.avail_out < d->binsz) { - zlib_output(d); - } - } - } - return res; -} - - - -static void* zlib_alloc(void* data, unsigned int items, unsigned int size) -{ - return (void*) driver_alloc(items*size); -} - -static void zlib_free(void* data, void* addr) -{ - driver_free(addr); -} - -#if defined(__APPLE__) && defined(__MACH__) && defined(HAVE_ZLIB_INFLATEGETDICTIONARY) - -/* Work around broken build system with runtime version test */ -static int have_inflateGetDictionary; - -static int zlib_init() -{ - unsigned int v[4] = {0, 0, 0, 0}; - unsigned hexver; - - sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]); - - hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3]; - - have_inflateGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */ - - return 0; -} -#else /* trust configure got it right */ -# ifdef HAVE_ZLIB_INFLATEGETDICTIONARY -# define have_inflateGetDictionary 1 -# else -# define have_inflateGetDictionary 0 -# endif -static int zlib_init() -{ - return 0; -} -#endif - -static ErlDrvData zlib_start(ErlDrvPort port, char* buf) -{ - ZLibData* d; - - if ((d = (ZLibData*) driver_alloc(sizeof(ZLibData))) == NULL) - return ERL_DRV_ERROR_GENERAL; - - memset(&d->s, 0, sizeof(z_stream)); - - d->s.zalloc = zlib_alloc; - d->s.zfree = zlib_free; - d->s.opaque = d; - d->s.data_type = Z_BINARY; - - d->port = port; - d->state = ST_NONE; - d->bin = NULL; - d->binsz = 0; - d->binsz_need = DEFAULT_BUFSZ; - d->crc = crc32(0L, Z_NULL, 0); - d->inflate_eos_seen = 0; - d->want_crc = 0; - return (ErlDrvData)d; -} - - -static void zlib_stop(ErlDrvData e) -{ - ZLibData* d = (ZLibData*)e; - - if (d->state == ST_DEFLATE) - deflateEnd(&d->s); - else if (d->state == ST_INFLATE) - inflateEnd(&d->s); - - if (d->bin != NULL) - driver_free_binary(d->bin); - - driver_free(d); -} - -static void zlib_flush(ErlDrvData drv_data) -{ - ZLibData* d = (ZLibData*) drv_data; - - driver_deq(d->port, driver_sizeq(d->port)); -} - -static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, - ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) -{ - ZLibData* d = (ZLibData*)drv_data; - int res; - - switch(command) { - case DEFLATE_INIT: - if (len != 4) goto badarg; - if (d->state != ST_NONE) goto badarg; - res = deflateInit(&d->s, i32(buf)); - if (res == Z_OK) { - d->state = ST_DEFLATE; - d->want_crc = 0; - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - - case DEFLATE_INIT2: { - int wbits; - - if (len != 20) goto badarg; - if (d->state != ST_NONE) goto badarg; - wbits = i32(buf+8); - res = deflateInit2(&d->s, i32(buf), i32(buf+4), wbits, - i32(buf+12), i32(buf+16)); - if (res == Z_OK) { - d->state = ST_DEFLATE; - d->want_crc = (wbits < 0); - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - } - - case DEFLATE_SETDICT: - if (d->state != ST_DEFLATE) goto badarg; - res = deflateSetDictionary(&d->s, (unsigned char*)buf, len); - if (res == Z_OK) { - return zlib_value(d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case DEFLATE_RESET: - if (len != 0) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = deflateReset(&d->s); - return zlib_return(res, rbuf, rlen); - - case DEFLATE_END: - if (len != 0) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = deflateEnd(&d->s); - d->state = ST_NONE; - return zlib_return(res, rbuf, rlen); - - case DEFLATE_PARAMS: - if (len != 8) goto badarg; - if (d->state != ST_DEFLATE) goto badarg; - res = deflateParams(&d->s, i32(buf), i32(buf+4)); - return zlib_return(res, rbuf, rlen); - - case DEFLATE: - if (d->state != ST_DEFLATE) goto badarg; - if (len != 4) goto badarg; - res = zlib_deflate(d, i32(buf)); - return zlib_return(res, rbuf, rlen); - - case INFLATE_INIT: - if (len != 0) goto badarg; - if (d->state != ST_NONE) goto badarg; - res = inflateInit(&d->s); - if (res == Z_OK) { - d->state = ST_INFLATE; - d->inflate_eos_seen = 0; - d->want_crc = 0; - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_INIT2: { - int wbits; - - if (len != 4) goto badarg; - if (d->state != ST_NONE) goto badarg; - wbits = i32(buf); - res = inflateInit2(&d->s, wbits); - if (res == Z_OK) { - d->state = ST_INFLATE; - d->inflate_eos_seen = 0; - d->want_crc = (wbits < 0); - d->crc = crc32(0L, Z_NULL, 0); - } - return zlib_return(res, rbuf, rlen); - } - - case INFLATE_SETDICT: - if (d->state != ST_INFLATE) goto badarg; - res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); - return zlib_return(res, rbuf, rlen); - - case INFLATE_GETDICT: - if (have_inflateGetDictionary) { - if (d->state != ST_INFLATE) goto badarg; - res = zlib_inflate_get_dictionary(d); - } else { - errno = ENOTSUP; - res = Z_ERRNO; - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_SYNC: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - if (driver_sizeq(d->port) == 0) { - res = Z_BUF_ERROR; - } else { - int vlen; - SysIOVec* iov = driver_peekq(d->port, &vlen); - - d->s.next_in = iov[0].iov_base; - d->s.avail_in = iov[0].iov_len; - res = inflateSync(&d->s); - } - return zlib_return(res, rbuf, rlen); - - case INFLATE_RESET: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = inflateReset(&d->s); - d->inflate_eos_seen = 0; - return zlib_return(res, rbuf, rlen); - - case INFLATE_END: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - driver_deq(d->port, driver_sizeq(d->port)); - res = inflateEnd(&d->s); - if (res == Z_OK && d->inflate_eos_seen == 0) { - res = Z_DATA_ERROR; - } - d->state = ST_NONE; - return zlib_return(res, rbuf, rlen); - - case INFLATE: - if (d->state != ST_INFLATE) goto badarg; - if (len != 4) goto badarg; - res = zlib_inflate(d, i32(buf)); - if (res == Z_NEED_DICT) { - return zlib_value2(3, d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case INFLATE_CHUNK: - if (d->state != ST_INFLATE) goto badarg; - if (len != 0) goto badarg; - res = zlib_inflate_chunk(d); - if (res == INFLATE_HAS_MORE) { - return zlib_value2(4, 0, rbuf, rlen); - } else if (res == Z_NEED_DICT) { - return zlib_value2(3, d->s.adler, rbuf, rlen); - } else { - return zlib_return(res, rbuf, rlen); - } - - case GET_QSIZE: - return zlib_value(driver_sizeq(d->port), rbuf, rlen); - - case GET_BUFSZ: - return zlib_value(d->binsz_need, rbuf, rlen); - - case SET_BUFSZ: { - int need; - if (len != 4) goto badarg; - need = i32(buf); - if ((need < 16) || (need > 0x00ffffff)) - goto badarg; - if (d->binsz_need != need) { - d->binsz_need = need; - if (d->bin != NULL) { - if (d->s.avail_out == d->binsz) { - driver_free_binary(d->bin); - d->bin = NULL; - d->binsz = 0; - } - else - zlib_output(d); - } - } - return zlib_return(Z_OK, rbuf, rlen); - } - - case CRC32_0: - return zlib_value(d->crc, rbuf, rlen); - - case CRC32_1: { - uLong crc = crc32(0L, Z_NULL, 0); - crc = crc32(crc, (unsigned char*) buf, len); - return zlib_value(crc, rbuf, rlen); - } - - case CRC32_2: { - uLong crc; - if (len < 4) goto badarg; - crc = (unsigned int) i32(buf); - crc = crc32(crc, (unsigned char*) buf+4, len-4); - return zlib_value(crc, rbuf, rlen); - } - - case ADLER32_1: { - uLong adler = adler32(0L, Z_NULL, 0); - adler = adler32(adler, (unsigned char*) buf, len); - return zlib_value(adler, rbuf, rlen); - } - - case ADLER32_2: { - uLong adler; - if (len < 4) goto badarg; - adler = (unsigned int) i32(buf); - adler = adler32(adler, (unsigned char*) buf+4, len-4); - return zlib_value(adler, rbuf, rlen); - } - - case CRC32_COMBINE: { - uLong crc, crc1, crc2, len2; - if (len != 12) goto badarg; - crc1 = (unsigned int) i32(buf); - crc2 = (unsigned int) i32(buf+4); - len2 = (unsigned int) i32(buf+8); - crc = crc32_combine(crc1, crc2, len2); - return zlib_value(crc, rbuf, rlen); - } - - case ADLER32_COMBINE: { - uLong adler, adler1, adler2, len2; - if (len != 12) goto badarg; - adler1 = (unsigned int) i32(buf); - adler2 = (unsigned int) i32(buf+4); - len2 = (unsigned int) i32(buf+8); - adler = adler32_combine(adler1, adler2, len2); - return zlib_value(adler, rbuf, rlen); - } - } - - badarg: - errno = EINVAL; - return zlib_return(Z_ERRNO, rbuf, rlen); -} - - - -static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev) -{ - ZLibData* d = (ZLibData*) drv_data; - - driver_enqv(d->port, ev, 0); -} diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index e425b99f16..7355df6059 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -108,16 +108,15 @@ static int lbuf_size = BUFSIZ; static Uint32 *lbuf; /* The current line buffer */ static int llen; /* The current line length */ static int lpos; /* The current "cursor position" in the line buffer */ - + /* NOTE: not the same as column position a char may not take a" + * column to display or it might take many columns + */ /* * Tags used in line buffer to show that these bytes represent special characters, * Max unicode is 0x0010ffff, so we have lots of place for meta tags... */ #define CONTROL_TAG 0x10000000U /* Control character, value in first position */ #define ESCAPED_TAG 0x01000000U /* Escaped character, value in first position */ -#ifdef HAVE_WCWIDTH -#define WIDE_TAG 0x02000000U /* Wide character, value in first position */ -#endif #define TAG_MASK 0xFF000000U #define MAXSIZE (1 << 16) @@ -156,6 +155,8 @@ static int insert_buf(byte*,int); static int write_buf(Uint32 *,int); static int outc(int c); static int move_cursor(int,int); +static int cp_pos_to_col(int cp_pos); + /* Termcap functions. */ static int start_termcap(void); @@ -891,8 +892,8 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd) tpos = 0; } } - } else { - DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d\n", (int)(SWord)fd, i)); + } else if (errno != EAGAIN && errno != EWOULDBLOCK) { + DEBUGLOG(("ttysl_from_tty: driver failure in read(%d,..) = %d (errno = %d)\n", (int)(SWord)fd, i, errno)); driver_failure(ttysl_port, -1); } } @@ -935,10 +936,10 @@ static int put_chars(byte *s, int l) int n; n = insert_buf(s, l); + if (lpos > llen) + llen = lpos; if (n > 0) write_buf(lbuf + lpos - n, n); - if (lpos > llen) - llen = lpos; return TRUE; } @@ -991,34 +992,36 @@ static int del_chars(int n) { int i, l, r; int pos; + int gcs; /* deleted grapheme characters */ update_cols(); /* Step forward or backwards over n logical characters. */ pos = step_over_chars(n); - + DEBUGLOG(("del_chars: %d from %d %d %d\n", n, lpos, pos, llen)); if (pos > lpos) { l = pos - lpos; /* Buffer characters to delete */ r = llen - lpos - l; /* Characters after deleted */ + gcs = cp_pos_to_col(pos) - cp_pos_to_col(lpos); /* Fix up buffer and buffer pointers. */ if (r > 0) memmove(lbuf + lpos, lbuf + pos, r * sizeof(Uint32)); llen -= l; /* Write out characters after, blank the tail and jump back to lpos. */ write_buf(lbuf + lpos, r); - for (i = l ; i > 0; --i) + for (i = gcs ; i > 0; --i) outc(' '); - if (COL(llen+l) == 0 && xn) + if (xn && COL(cp_pos_to_col(llen)+gcs) == 0) { outc(' '); move_left(1); } - move_cursor(llen + l, lpos); + move_cursor(llen + gcs, lpos); } else if (pos < lpos) { l = lpos - pos; /* Buffer characters */ r = llen - lpos; /* Characters after deleted */ - move_cursor(lpos, lpos-l); /* Move back */ + gcs = -move_cursor(lpos, lpos-l); /* Move back */ /* Fix up buffer and buffer pointers. */ if (r > 0) memmove(lbuf + pos, lbuf + lpos, r * sizeof(Uint32)); @@ -1026,14 +1029,14 @@ static int del_chars(int n) llen -= l; /* Write out characters after, blank the tail and jump back to lpos. */ write_buf(lbuf + lpos, r); - for (i = l ; i > 0; --i) - outc(' '); - if (COL(llen+l) == 0 && xn) + for (i = gcs ; i > 0; --i) + outc(' '); + if (xn && COL(cp_pos_to_col(llen)+gcs) == 0) { - outc(' '); - move_left(1); + outc(' '); + move_left(1); } - move_cursor(llen + l, lpos); + move_cursor(llen + gcs, lpos); } return TRUE; } @@ -1047,22 +1050,12 @@ static int step_over_chars(int n) end = lbuf + llen; c = lbuf + lpos; for ( ; n > 0 && c < end; --n) { -#ifdef HAVE_WCWIDTH - while (*c & WIDE_TAG) { - c++; - } -#endif c++; while (c < end && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) c++; } for ( ; n < 0 && c > beg; n++) { --c; -#ifdef HAVE_WCWIDTH - while (c > beg + 1 && (c[-1] & WIDE_TAG)) { - --c; - } -#endif while (c > beg && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) --c; } @@ -1088,15 +1081,6 @@ static int insert_buf(byte *s, int n) ++pos; } if ((utf8_mode && (ch >= 128 || isprint(ch))) || (ch <= 255 && isprint(ch))) { -#ifdef HAVE_WCWIDTH - int width; - if ((width = wcwidth(ch)) > 1) { - while (--width) { - DEBUGLOG(("insert_buf: Wide(UTF-8):%d,%d",width,ch)); - lbuf[lpos++] = (WIDE_TAG | ((Uint32) ch)); - } - } -#endif DEBUGLOG(("insert_buf: Printable(UTF-8):%d",ch)); lbuf[lpos++] = (Uint32) ch; } else if (ch >= 128) { /* not utf8 mode */ @@ -1110,15 +1094,14 @@ static int insert_buf(byte *s, int n) lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); ch = 0; } while (lpos % 8); - } else if (ch == '\e' || ch == '\n' || ch == '\r') { + } else if (ch == '\e') { + DEBUGLOG(("insert_buf: ANSI Escape: \\e")); + lbuf[lpos++] = (CONTROL_TAG | ((Uint32) ch)); + } else if (ch == '\n' || ch == '\r') { write_buf(lbuf + buffpos, lpos - buffpos); - if (ch == '\e') { - outc('\e'); - } else { outc('\r'); if (ch == '\n') outc('\n'); - } if (llen > lpos) { memcpy(lbuf, lbuf + lpos, llen - lpos); } @@ -1166,14 +1149,17 @@ static int write_buf(Uint32 *s, int n) } --n; ++s; - } - else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { + } else if (*s == (CONTROL_TAG | ((Uint32) '\t'))) { outc(lastput = ' '); --n; s++; while (n > 0 && *s == CONTROL_TAG) { outc(lastput = ' '); --n; s++; } + } else if (*s == (CONTROL_TAG | ((Uint32) '\e'))) { + outc(lastput = '\e'); + --n; + ++s; } else if (*s & CONTROL_TAG) { outc('^'); outc(lastput = ((byte) ((*s == 0177) ? '?' : *s | 0x40))); @@ -1204,10 +1190,6 @@ static int write_buf(Uint32 *s, int n) if (octbuff != octtmp) { driver_free(octbuff); } -#ifdef HAVE_WCWIDTH - } else if (*s & WIDE_TAG) { - --n; s++; -#endif } else { DEBUGLOG(("write_buf: Very unexpected character %d",(int) *s)); ++n; @@ -1216,7 +1198,7 @@ static int write_buf(Uint32 *s, int n) } /* Check landed in first column of new line and have 'xn' bug. */ n = s - lbuf; - if (COL(n) == 0 && xn && n != 0) { + if (xn && n != 0 && COL(cp_pos_to_col(n)) == 0) { if (n >= llen) { outc(' '); } else if (lastput == 0) { /* A multibyte UTF8 character */ @@ -1246,14 +1228,19 @@ static int outc(int c) return 1; } -static int move_cursor(int from, int to) +static int move_cursor(int from_pos, int to_pos) { + int from_col, to_col; int dc, dl; - update_cols(); - dc = COL(to) - COL(from); - dl = LINE(to) - LINE(from); + from_col = cp_pos_to_col(from_pos); + to_col = cp_pos_to_col(to_pos); + + dc = COL(to_col) - COL(from_col); + dl = LINE(to_col) - LINE(from_col); + DEBUGLOG(("move_cursor: from %d %d to %d %d => %d %d\n", + from_pos, from_col, to_pos, to_col, dl, dc)); if (dl > 0) move_down(dl); else if (dl < 0) @@ -1262,7 +1249,66 @@ static int move_cursor(int from, int to) move_right(dc); else if (dc < 0) move_left(-dc); - return TRUE; + return to_col-from_col; +} + +/* + * Returns the length of an ANSI escape code in a buffer, this function only consider + * color escape sequences like `\e[33m` or `\e[21;33m`. If a sequence has no valid + * terminator, the length is equal the number of characters between `\e` and the first + * invalid character, inclusive. + */ + +static int ansi_escape_width(Uint32 *s, int max_length) +{ + int i; + + if (*s != (CONTROL_TAG | ((Uint32) '\e'))) { + return 0; + } else if (max_length <= 1) { + return 1; + } else if (s[1] != '[') { + return 2; + } + + for (i = 2; i < max_length && (s[i] == ';' || (s[i] >= '0' && s[i] <= '9')); i++); + + return i + 1; +} + +static int cp_pos_to_col(int cp_pos) +{ + /* + * If we don't have any character width information. Assume that + * code points are one column wide + */ + int w = 1; + int col = 0; + int i = 0; + int j; + + if (cp_pos > llen) { + col += cp_pos - llen; + cp_pos = llen; + } + + while (i < cp_pos) { + j = ansi_escape_width(lbuf + i, llen - i); + + if (j > 0) { + i += j; + } else { +#ifdef HAVE_WCWIDTH + w = wcwidth(lbuf[i]); +#endif + if (w > 0) { + col += w; + } + i++; + } + } + + return col; } static int start_termcap(void) diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4 index dca3887564..b3c9a460bb 100644 --- a/erts/emulator/hipe/hipe_amd64_bifs.m4 +++ b/erts/emulator/hipe/hipe_amd64_bifs.m4 @@ -39,7 +39,7 @@ define(HANDLE_GOT_MBUF,` 3: call nbif_$1_gc_after_bif /* `HANDLE_GOT_MBUF' */ jmp 2b') -`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +`#if defined(ERTS_ENABLE_LOCK_CHECK) # define CALL_BIF(F) \ movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \ movq %r11, P_BIF_CALLEE(P); \ diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4 index a9097dabde..554faa2567 100644 --- a/erts/emulator/hipe/hipe_arm_bifs.m4 +++ b/erts/emulator/hipe/hipe_arm_bifs.m4 @@ -29,7 +29,7 @@ include(`hipe/hipe_arm_asm.m4') .p2align 2 .arm -`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +`#if defined(ERTS_ENABLE_LOCK_CHECK) # define CALL_BIF(F) ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper #else # define CALL_BIF(F) bl nbif_impl_##F diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 0225f17613..380031bf13 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -53,8 +53,6 @@ #include "hipe_literals.h" #endif -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - int term_to_Sint32(Eterm term, Sint *sp) { @@ -615,7 +613,7 @@ static ErtsCodeInfo* hipe_find_emu_address(Eterm mod, Eterm name, unsigned int a n = code_hdr->num_functions; for (i = 0; i < n; ++i) { ErtsCodeInfo *ci = code_hdr->functions[i]; - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); if (ci->mfa.function == name && ci->mfa.arity == arity) return ci; } @@ -1000,7 +998,7 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); fe->native_address = native_address; - if (erts_smp_refc_dectest(&fe->refc, 0) == 0) + if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); BIF_RET(am_true); } @@ -1048,7 +1046,7 @@ static struct { * they create a new stub for the mfa, which forces locking. * XXX: Redesign apply et al to avoid those updates. */ - erts_smp_rwmtx_t lock; + erts_rwmtx_t lock; } hipe_mfa_info_table; Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ @@ -1129,27 +1127,28 @@ struct hipe_ref { static inline void hipe_mfa_info_table_init_lock(void) { - erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); + erts_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } static inline void hipe_mfa_info_table_rlock(void) { - erts_smp_rwmtx_rlock(&hipe_mfa_info_table.lock); + erts_rwmtx_rlock(&hipe_mfa_info_table.lock); } static inline void hipe_mfa_info_table_runlock(void) { - erts_smp_rwmtx_runlock(&hipe_mfa_info_table.lock); + erts_rwmtx_runlock(&hipe_mfa_info_table.lock); } static inline void hipe_mfa_info_table_rwlock(void) { - erts_smp_rwmtx_rwlock(&hipe_mfa_info_table.lock); + erts_rwmtx_rwlock(&hipe_mfa_info_table.lock); } static inline void hipe_mfa_info_table_rwunlock(void) { - erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); + erts_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } static ERTS_INLINE @@ -1635,7 +1634,7 @@ void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, { struct hipe_ref* ref = first_ref; - ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking()); while (ref) { struct hipe_ref* free_ref = ref; @@ -1681,9 +1680,9 @@ void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, { struct hipe_sdesc* sdesc = first_sdesc; - ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking()); - ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + ERTS_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ while (sdesc) { struct hipe_sdesc* free_sdesc = sdesc; @@ -1701,7 +1700,7 @@ void hipe_purge_module(Module* modp, int is_blocking) { ASSERT(modp); - ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(is_blocking == erts_thr_progress_is_blocking()); DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); @@ -1710,7 +1709,7 @@ void hipe_purge_module(Module* modp, int is_blocking) * Remove all hipe_ref's (external calls) from the old module instance */ if (modp->old.hipe_code->first_hipe_ref) { - ERTS_SMP_LC_ASSERT(is_blocking); + ERTS_LC_ASSERT(is_blocking); hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, make_atom(modp->module), is_blocking); @@ -1721,7 +1720,7 @@ void hipe_purge_module(Module* modp, int is_blocking) * Remove all hipe_sdesc's for the old module instance */ if (modp->old.hipe_code->first_hipe_sdesc) { - ERTS_SMP_LC_ASSERT(is_blocking); + ERTS_LC_ASSERT(is_blocking); hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, make_atom(modp->module), is_blocking); @@ -1772,7 +1771,7 @@ void hipe_redirect_to_module(Module* modp) struct hipe_mfa_info *p; struct hipe_ref_head* refh; - ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { if (p->new_address) { diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 3d3df4fd48..73d07f0ce5 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -32,11 +32,10 @@ #include "big.h" #include "error.h" #include "beam_load.h" +#include "erl_vm.h" #include "hipe_bif0.h" #include "hipe_bif1.h" -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) { ErtsCodeInfo *ci; @@ -46,17 +45,17 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] == BeamOpCode(op_hipe_trap_call)) + if (BeamIsOpCode(pc[0], op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); - if (pc[0] == BeamOpCode(op_hipe_call_count)) + if (BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(NIL); hcc = erts_alloc(ERTS_ALC_T_HIPE_SL, sizeof(*hcc)); hcc->count = 0; hcc->opcode = pc[0]; ci->u.hcc = hcc; - pc[0] = BeamOpCode(op_hipe_call_count); + pc[0] = BeamOpCodeAddr(op_hipe_call_count); BIF_RET(am_true); } @@ -70,9 +69,9 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; count = hcc->count; @@ -91,9 +90,9 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; BIF_RET(make_small(hcc->count)); @@ -109,9 +108,9 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) ci = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!ci) BIF_ERROR(BIF_P, BADARG); - ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(BeamIsOpCode(ci->op, op_i_func_info_IaaI)); pc = erts_codeinfo_to_code(ci); - if (pc[0] != BeamOpCode(op_hipe_call_count)) + if (! BeamIsOpCode(pc[0], op_hipe_call_count)) BIF_RET(am_false); hcc = ci->u.hcc; count = hcc->count; diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index e04d3d32d1..9ebbb22846 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -45,7 +45,7 @@ static void proc_unlock(Process* c_p, Process* rp) locks &= ~ERTS_PROC_LOCK_MAIN; } if (rp && locks) { - erts_smp_proc_unlock(rp, locks); + erts_proc_unlock(rp, locks); } } @@ -153,14 +153,14 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0) BIF_RET(am_true); } -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_CHECK) BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1); -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ +# define ERTS_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ __FILE__, __LINE__) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1) @@ -168,13 +168,13 @@ BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1) typedef BIF_RETTYPE nBif(NBIF_ALIST_1); nBif* fp = (nBif*) (BIF_P->hipe.bif_callee); BIF_RETTYPE res; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P); + ERTS_UNREQ_PROC_MAIN_LOCK(BIF_P); res = (*fp)(NBIF_CALL_ARGS); - ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P); + ERTS_REQ_PROC_MAIN_LOCK(BIF_P); return res; } -#endif /* ERTS_ENABLE_LOCK_CHECK && ERTS_SMP */ +#endif /* ERTS_ENABLE_LOCK_CHECK*/ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2) diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index f034c4700c..bebe20a18e 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -262,18 +262,12 @@ noproc_primop_interface_2(nbif_bs_get_utf16, erts_bs_get_utf16) noproc_primop_interface_2(nbif_bs_validate_unicode_retract, hipe_bs_validate_unicode_retract) /* - * Bit-syntax primops. The ERTS_SMP runtime system requires P, + * Bit-syntax primops. The runtime system requires P, * hence the use of nocons_nofail_primop_interface_N(). - * When ERTS_SMP is disabled, noproc_primop_interface_N() - * should be used instead. */ nocons_nofail_primop_interface_5(nbif_bs_put_small_float, hipe_bs_put_small_float) noproc_primop_interface_5(nbif_bs_put_bits, hipe_bs_put_bits) -ifelse(ERTS_SMP,1,` nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer) -',` -noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer) -')dnl nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg) @@ -283,13 +277,8 @@ nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe) noproc_primop_interface_1(nbif_emasculate_binary, hipe_emasculate_binary) -/* - * SMP-specific stuff - */ -ifelse(ERTS_SMP,1,` nocons_nofail_primop_interface_0(nbif_clear_timeout, hipe_clear_timeout) noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc) -',)dnl /* * BIFs that disable GC while trapping are called via a wrapper diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab new file mode 100644 index 0000000000..a01baebddf --- /dev/null +++ b/erts/emulator/hipe/hipe_instrs.tab @@ -0,0 +1,141 @@ +// -*- c -*- +// +// %CopyrightBegin% +// +// Copyright Ericsson AB 2017. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// %CopyrightEnd% +// + + +HIPE_MODE_SWITCH(Cmd) { + SWAPOUT; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + c_p->fcalls = FCALLS; + c_p->def_arg_reg[4] = -neg_o_reds; + c_p = hipe_mode_switch(c_p, $Cmd, reg); +} + +hipe_trap_call := hipe_trap.call.post; +hipe_trap_call_closure := hipe_trap.call_closure.post; +hipe_trap_return := hipe_trap.return.post; +hipe_trap_throw := hipe_trap.throw.post; +hipe_trap_resume := hipe_trap.resume.post; + +hipe_trap.call() { + /* + * I[-5]: &&lb_i_func_info_IaaI + * I[-4]: Native code callee (inserted by HiPE) + * I[-3]: Module (tagged atom) + * I[-2]: Function (tagged atom) + * I[-1]: Arity (untagged integer) + * I[ 0]: &&lb_hipe_trap_call + * ... remainder of original BEAM code + */ + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); + c_p->hipe.u.ncallee = ci->u.ncallee; + ++hipe_trap_count; + $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); +} + +hipe_trap.call_closure() { + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); + c_p->hipe.u.ncallee = ci->u.ncallee; + ++hipe_trap_count; + $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); +} + +hipe_trap.return() { + $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); +} + +hipe_trap.throw() { + $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_THROW); +} + +hipe_trap.resume() { + $HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RESUME); +} + +hipe_trap.post() { +#ifdef DEBUG + pid = c_p->common.id; /* may have switched process... */ +#endif + reg = erts_proc_sched_data(c_p)->x_reg_array; + freg = erts_proc_sched_data(c_p)->f_reg_array; + ERL_BITS_RELOAD_STATEP(c_p); + /* XXX: this abuse of def_arg_reg[] is horrid! */ + neg_o_reds = -c_p->def_arg_reg[4]; + FCALLS = c_p->fcalls; + SWAPIN; + ERTS_DBG_CHK_REDS(c_p, FCALLS); + switch( c_p->def_arg_reg[3] ) { + case HIPE_MODE_SWITCH_RES_RETURN: + ASSERT(is_value(reg[0])); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); + case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: + c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; + /*fall through*/ + case HIPE_MODE_SWITCH_RES_CALL_BEAM: + SET_I(c_p->i); + Dispatch(); + case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: + /* This can be used to call any function value, but currently + it's only used to call closures referring to unloaded + modules. */ + { + BeamInstr *next; + + next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); + HEAVY_SWAPIN; + if (next != NULL) { + SET_I(next); + Dispatchfun(); + } + goto find_func_info; + } + case HIPE_MODE_SWITCH_RES_THROW: + c_p->cp = NULL; + I = handle_error(c_p, I, reg, NULL); + goto post_error_handling; + default: + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + } + //| -no_next; +} + +hipe_call_count() { + /* + * I[-5]: &&lb_i_func_info_IaaI + * I[-4]: pointer to struct hipe_call_count (inserted by HiPE) + * I[-3]: Module (tagged atom) + * I[-2]: Function (tagged atom) + * I[-1]: Arity (untagged integer) + * I[ 0]: &&lb_hipe_call_count + * ... remainder of original BEAM code + */ + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = ci->u.hcc; + ASSERT(IsOpCode(ci->op, i_func_info_IaaI)); + ASSERT(hcc != NULL); + ASSERT(VALID_INSTR(hcc->opcode)); + ++(hcc->count); + Goto(hcc->opcode); + //| -no_next; +} diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 4573980e1e..6ea120c65c 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -441,9 +441,7 @@ static const struct rts_param rts_params[] = { { 11, "ERL_FUN_SIZE", 1, ERL_FUN_SIZE }, { 12, "P_SCHED_DATA", -#ifdef ERTS_SMP 1, offsetof(struct process, scheduler_data) -#endif }, { 14, "P_FP_EXCEPTION", #if !defined(NO_FPE_SIGNALS) || defined(HIPE) @@ -453,11 +451,7 @@ static const struct rts_param rts_params[] = { /* This flag is always defined, but its value is configuration-dependent. */ { 15, "ERTS_IS_SMP", 1, -#if defined(ERTS_SMP) 1 -#else - 0 -#endif }, /* This flag is always defined, but its value is configuration-dependent. */ { 16, "ERTS_NO_FPE_SIGNALS", @@ -513,7 +507,7 @@ static const struct rts_param rts_params[] = { #endif }, { 48, "P_BIF_CALLEE", -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_CHECK) 1, offsetof(struct process, hipe.bif_callee) #endif }, diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index ba7ae1e6a8..8b497c9970 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -36,15 +36,15 @@ #include "hipe_stack.h" #include "hipe_bif0.h" /* hipe_mfa_info_table_init() */ -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ +#if defined(ERTS_ENABLE_LOCK_CHECK) +# define ERTS_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ __FILE__, __LINE__) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) #else -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) +# define ERTS_REQ_PROC_MAIN_LOCK(P) +# define ERTS_UNREQ_PROC_MAIN_LOCK(P) #endif @@ -155,8 +155,6 @@ void hipe_check_pcb(Process *p, const char *file, unsigned line) #include "hipe_arm_glue.h" #endif -#define BeamOpCode(Op) ((Uint)BeamOp(Op)) - Uint hipe_beam_pc_return[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_throw[1]; /* needed in hipe_debug.c */ Uint hipe_beam_pc_resume[1]; /* needed by hipe_set_timeout() */ @@ -166,9 +164,9 @@ void hipe_mode_switch_init(void) { hipe_arch_glue_init(); - hipe_beam_pc_return[0] = BeamOpCode(op_hipe_trap_return); - hipe_beam_pc_throw[0] = BeamOpCode(op_hipe_trap_throw); - hipe_beam_pc_resume[0] = BeamOpCode(op_hipe_trap_resume); + hipe_beam_pc_return[0] = BeamOpCodeAddr(op_hipe_trap_return); + hipe_beam_pc_throw[0] = BeamOpCodeAddr(op_hipe_trap_throw); + hipe_beam_pc_resume[0] = BeamOpCodeAddr(op_hipe_trap_resume); hipe_beam_catch_throw = make_catch(beam_catches_cons(hipe_beam_pc_throw, BEAM_CATCHES_NIL)); @@ -182,8 +180,8 @@ void hipe_set_call_trap(ErtsCodeInfo* ci, void *nfun, int is_closure) HIPE_ASSERT(ci->op == BeamOpCode(op_i_func_info_IaaI)); bfun[0] = is_closure - ? BeamOpCode(op_hipe_trap_call_closure) - : BeamOpCode(op_hipe_trap_call); + ? BeamOpCodeAddr(op_hipe_trap_call_closure) + : BeamOpCodeAddr(op_hipe_trap_call); ci->u.ncallee = (void (*)(void)) nfun; } @@ -394,7 +392,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) goto do_schedule; } - if (!(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) { + if (!(erts_atomic32_read_acqb(&p->state) & ERTS_PSFLG_ACTIVE)) { for (i = 0; i < p->arity; ++i) p->arg_reg[i] = reg[i]; goto do_schedule; @@ -490,19 +488,17 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) case HIPE_MODE_SWITCH_RES_WAIT: case HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT: { /* same semantics, different debug trace messages */ -#ifdef ERTS_SMP /* XXX: BEAM has different entries for the locked and unlocked cases. HiPE doesn't, so we must check dynamically. */ if (p->hipe_smp.have_receive_locks) p->hipe_smp.have_receive_locks = 0; else - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); -#endif + erts_proc_lock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); p->i = hipe_beam_pc_resume; p->arity = 0; - erts_smp_atomic32_read_band_relb(&p->state, + erts_atomic32_read_band_relb(&p->state, ~ERTS_PSFLG_ACTIVE); - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); do_schedule: { struct saved_calls *scb; @@ -513,21 +509,19 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) /* The process may have died while it was executing, if so we return out from native code to the interpreter */ - if (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) + if (erts_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) p->i = beam_exit; #ifdef DEBUG ASSERT(p->debug_reds_in == reds_in); #endif p->flags &= ~F_HIPE_MODE; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p); + ERTS_UNREQ_PROC_MAIN_LOCK(p); p = erts_schedule(NULL, p, reds_in - p->fcalls); - ERTS_SMP_REQ_PROC_MAIN_LOCK(p); + ERTS_REQ_PROC_MAIN_LOCK(p); ASSERT(!(p->flags & F_HIPE_MODE)); -#ifdef ERTS_SMP p->hipe_smp.have_receive_locks = 0; reg = p->scheduler_data->x_reg_array; -#endif } { Eterm *argp; @@ -651,10 +645,10 @@ void hipe_inc_nstack(Process *p) p->hipe.nsp = new_nstack + (p->hipe.nsp - old_nstack); p->hipe.nstack = new_nstack; if (p->hipe.nstgraylim) - p->hipe.nstgraylim = + p->hipe.nstgraylim = new_nstack + (p->hipe.nstgraylim - old_nstack); if (p->hipe.nstblacklim) - p->hipe.nstblacklim = + p->hipe.nstblacklim = new_nstack + (p->hipe.nstblacklim - old_nstack); } } diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index d8044fe6da..23f64a6991 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -143,12 +143,10 @@ BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1) else { int tres = erts_set_proc_timer_term(p, timeout_value); if (tres != 0) { /* Wrong time */ -#ifdef ERTS_SMP if (p->hipe_smp.have_receive_locks) { p->hipe_smp.have_receive_locks = 0; - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE); } -#endif BIF_ERROR(p, EXC_TIMEOUT_VALUE); } } @@ -335,9 +333,7 @@ Binary *hipe_bs_reallocate(Binary* oldbptr, int newsize) } int hipe_bs_put_big_integer( -#ifdef ERTS_SMP Process *p, -#endif Eterm arg, Uint num_bits, byte* base, unsigned offset, unsigned flags) { byte *save_bin_buf; @@ -530,26 +526,22 @@ Eterm hipe_check_get_msg(Process *c_p) msgp = PEEK_MESSAGE(c_p); if (!msgp) { -#ifdef ERTS_SMP - erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); /* Make sure messages wont pass exit signals... */ if (ERTS_PROC_PENDING_EXIT(c_p)) { - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); return THE_NON_VALUE; /* Will be rescheduled for exit */ } - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + ERTS_MSGQ_MV_INQ2PRIVQ(c_p); msgp = PEEK_MESSAGE(c_p); if (msgp) - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); else { /* XXX: BEAM doesn't need this */ c_p->hipe_smp.have_receive_locks = 1; -#endif c_p->flags &= ~F_DELAY_GC; return THE_NON_VALUE; -#ifdef ERTS_SMP } -#endif } if (is_non_value(ERL_MESSAGE_TERM(msgp)) @@ -573,7 +565,6 @@ Eterm hipe_check_get_msg(Process *c_p) /* * SMP-specific stuff */ -#ifdef ERTS_SMP /* * This is like the timeout BEAM instruction. @@ -584,14 +575,12 @@ void hipe_clear_timeout(Process *c_p) * A timeout has occurred. Reset the save pointer so that the next * receive statement will examine the first message first. */ -#ifdef ERTS_SMP /* XXX: BEAM has different entries for the locked and unlocked cases. HiPE doesn't, so we must check dynamically. */ if (c_p->hipe_smp.have_receive_locks) { c_p->hipe_smp.have_receive_locks = 0; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); + erts_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE); } -#endif if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) { trace_receive(c_p, am_clock_service, am_timeout, NULL); } @@ -601,7 +590,6 @@ void hipe_clear_timeout(Process *c_p) void hipe_atomic_inc(int *counter) { - erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter); + erts_atomic_inc_nob((erts_atomic_t*)counter); } -#endif diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h index 38f874888b..cbc7ab8dc6 100644 --- a/erts/emulator/hipe/hipe_native_bif.h +++ b/erts/emulator/hipe/hipe_native_bif.h @@ -107,11 +107,7 @@ void hipe_emasculate_binary(Eterm); /* * Stuff that is different in SMP and non-SMP. */ -#ifdef ERTS_SMP int hipe_bs_put_big_integer(Process*, Eterm, Uint, byte*, unsigned, unsigned); -#else -int hipe_bs_put_big_integer(Eterm, Uint, byte*, unsigned, unsigned); -#endif AEXTERN(Eterm,nbif_check_get_msg,(Process*)); Eterm hipe_check_get_msg(Process*); @@ -122,12 +118,10 @@ BIF_RETTYPE hipe_bifs_debug_native_called_2(BIF_ALIST_2); /* * SMP-specific stuff */ -#ifdef ERTS_SMP AEXTERN(void,nbif_atomic_inc,(void)); AEXTERN(void,nbif_clear_timeout,(Process*)); void hipe_atomic_inc(int*); void hipe_clear_timeout(Process*); -#endif #define BIF_LIST(M,F,A,B,C,I) AEXTERN(Eterm,nbif_##C,(void)); #include "erl_bif_list.h" diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab index 96e4c0da91..19a3820a6a 100644 --- a/erts/emulator/hipe/hipe_ops.tab +++ b/erts/emulator/hipe/hipe_ops.tab @@ -23,4 +23,7 @@ hipe_trap_call_closure hipe_trap_return hipe_trap_throw hipe_trap_resume + +%cold hipe_call_count +%hot diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4 index 79a8bef77d..283fbbb200 100644 --- a/erts/emulator/hipe/hipe_ppc_bifs.m4 +++ b/erts/emulator/hipe/hipe_ppc_bifs.m4 @@ -25,7 +25,7 @@ include(`hipe/hipe_ppc_asm.m4') #`include' "config.h" #`include' "hipe_literals.h" -`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +`#if defined(ERTS_ENABLE_LOCK_CHECK) # define CALL_BIF(F) STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) #else # define CALL_BIF(F) bl CSYM(nbif_impl_##F) diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h index 4fcbc9df38..6aac5e6205 100644 --- a/erts/emulator/hipe/hipe_primops.h +++ b/erts/emulator/hipe/hipe_primops.h @@ -41,10 +41,8 @@ PRIMOP_LIST(am_bnot, &nbif_bnot_1) PRIMOP_LIST(am_gc_1, &nbif_gc_1) PRIMOP_LIST(am_check_get_msg, &nbif_check_get_msg) -#ifdef ERTS_SMP PRIMOP_LIST(am_atomic_inc, &nbif_atomic_inc) PRIMOP_LIST(am_clear_timeout, &nbif_clear_timeout) -#endif PRIMOP_LIST(am_select_msg, &nbif_select_msg) PRIMOP_LIST(am_set_timeout, &nbif_set_timeout) PRIMOP_LIST(am_rethrow, &nbif_rethrow) diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h index cc92bf653c..ef14c75f6c 100644 --- a/erts/emulator/hipe/hipe_process.h +++ b/erts/emulator/hipe/hipe_process.h @@ -49,7 +49,7 @@ struct hipe_process_state { #ifdef NO_FPE_SIGNALS double float_result; /* to be checked for inf/NaN by hipe_emulate_fpe */ #endif -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +#if defined(ERTS_ENABLE_LOCK_CHECK) void (*bif_callee)(void); /* When calling BIF's via debug wrapper */ #endif #ifdef DEBUG @@ -82,7 +82,6 @@ static __inline__ void hipe_delete_process(struct hipe_process_state *p) erts_free(ERTS_ALC_T_HIPE_STK, (void*)p->nstack); } -#ifdef ERTS_SMP struct hipe_process_state_smp { int have_receive_locks; }; @@ -91,6 +90,5 @@ static __inline__ void hipe_init_process_smp(struct hipe_process_state_smp *p) { p->have_receive_locks = 0; } -#endif #endif /* HIPE_PROCESS_H */ diff --git a/erts/emulator/hipe/hipe_signal.h b/erts/emulator/hipe/hipe_signal.h index 5d8621135b..524def11a4 100644 --- a/erts/emulator/hipe/hipe_signal.h +++ b/erts/emulator/hipe/hipe_signal.h @@ -27,13 +27,9 @@ #if defined(__i386__) || defined(__x86_64__) extern void hipe_signal_init(void); -#else -static __inline__ void hipe_signal_init(void) { } -#endif - -#if defined(ERTS_SMP) && (defined(__i386__) || defined(__x86_64__)) extern void hipe_thread_signal_init(void); #else +static __inline__ void hipe_signal_init(void) { } static __inline__ void hipe_thread_signal_init(void) { } #endif diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4 index 14330c2f1c..1b49fa57fd 100644 --- a/erts/emulator/hipe/hipe_sparc_bifs.m4 +++ b/erts/emulator/hipe/hipe_sparc_bifs.m4 @@ -28,7 +28,7 @@ include(`hipe/hipe_sparc_asm.m4') .section ".text" .align 4 -`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +`#if defined(ERTS_ENABLE_LOCK_CHECK) # define CALL_BIF(F) set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper #else # define CALL_BIF(F) call nbif_impl_##F diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4 index aecf67dc1b..9cb343d067 100644 --- a/erts/emulator/hipe/hipe_x86_bifs.m4 +++ b/erts/emulator/hipe/hipe_x86_bifs.m4 @@ -31,7 +31,7 @@ include(`hipe/hipe_x86_asm.m4') #define TEST_GOT_EXN cmpl $THE_NON_VALUE,%eax #endif' -`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) +`#if defined(ERTS_ENABLE_LOCK_CHECK) # define CALL_BIF(F) movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) #else # define CALL_BIF(F) call CSYM(nbif_impl_##F) diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index be68d7d463..d3b6933155 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -45,10 +45,8 @@ #include <signal.h> #include <stdio.h> #include <stdlib.h> -#ifdef ERTS_SMP #include "sys.h" #include "erl_alloc.h" -#endif #include "hipe_signal.h" #if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) @@ -259,7 +257,6 @@ static void hipe_sigaltstack(void *ss_sp) } } -#ifdef ERTS_SMP /* * Set up alternate signal stack for an Erlang process scheduler thread. */ @@ -269,7 +266,6 @@ void hipe_thread_signal_init(void) We use it to suppress false leak report from valgrind */ hipe_sigaltstack(erts_alloc_permanent_cache_aligned(ERTS_ALC_T_HIPE_LL, SIGSTKSZ)); } -#endif /* * Set up alternate signal stack for the main thread, @@ -277,10 +273,6 @@ void hipe_thread_signal_init(void) */ static void hipe_sigaltstack_init(void) { -#if !defined(ERTS_SMP) - static unsigned long my_sigstack[SIGSTKSZ/sizeof(long)]; - hipe_sigaltstack(my_sigstack); -#endif } /* diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c new file mode 100644 index 0000000000..a9c5b05e47 --- /dev/null +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -0,0 +1,986 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#define STATIC_ERLANG_NIF 1 + +#include <stdio.h> +#include <zlib.h> + +#include "erl_nif.h" +#include "config.h" +#include "sys.h" + +#ifdef VALGRIND +# include <valgrind/memcheck.h> +#endif + +#define INFL_DICT_SZ (32768) + +/* NIF interface declarations */ +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info); +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); +static void unload(ErlNifEnv *env, void* priv_data); + +static ErlNifResourceType *rtype_zlib; + +static ERL_NIF_TERM am_not_on_controlling_process; + +static ERL_NIF_TERM am_not_initialized; +static ERL_NIF_TERM am_already_initialized; + +static ERL_NIF_TERM am_ok; +static ERL_NIF_TERM am_error; + +static ERL_NIF_TERM am_continue; +static ERL_NIF_TERM am_finished; + +static ERL_NIF_TERM am_not_supported; +static ERL_NIF_TERM am_need_dictionary; + +static ERL_NIF_TERM am_empty; + +static ERL_NIF_TERM am_stream_end; +static ERL_NIF_TERM am_stream_error; +static ERL_NIF_TERM am_data_error; +static ERL_NIF_TERM am_mem_error; +static ERL_NIF_TERM am_buf_error; +static ERL_NIF_TERM am_version_error; +static ERL_NIF_TERM am_unknown_error; + +typedef enum { + ST_NONE = 0, + ST_DEFLATE = 1, + ST_INFLATE = 2, + ST_CLOSED = 3 +} zlib_state_t; + +/* Controls what to do when the user attempts to decompress more data after + * Z_STREAM_END has been returned: + * + * - 'cut' wipes all further input and returns empty results until reset by + * the user. This is the default behavior, matching that of the old driver. + * - 'reset' resets the state without discarding any input, making it possible + * to decompress blindly concatenated streams. + * - 'error' crashes with a data error. */ +typedef enum { + EOS_BEHAVIOR_ERROR = 0, + EOS_BEHAVIOR_RESET = 1, + EOS_BEHAVIOR_CUT = 2 +} zlib_eos_behavior_t; + +typedef struct { + z_stream s; + zlib_state_t state; + + zlib_eos_behavior_t eos_behavior; + + /* These refer to the plaintext CRC, and are only needed for zlib:crc32/1 + * which is deprecated. */ + uLong input_crc; + uLong output_crc; + int want_input_crc; + int want_output_crc; + + int is_raw_stream; + + int eos_seen; + + /* DEPRECATED */ + int inflateChunk_buffer_size; + + ErlNifPid controlling_process; + + ErlNifIOQueue *input_queue; + + ErlNifEnv *stash_env; + ERL_NIF_TERM stash_term; +} zlib_data_t; + +/* The NIFs: */ + +static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); + +static ErlNifFunc nif_funcs[] = { + /* deflate */ + {"deflateInit_nif", 6, zlib_deflateInit}, + {"deflateSetDictionary_nif", 2, zlib_deflateSetDictionary}, + {"deflateReset_nif", 1, zlib_deflateReset}, + {"deflateEnd_nif", 1, zlib_deflateEnd}, + {"deflateParams_nif", 3, zlib_deflateParams}, + {"deflate_nif", 4, zlib_deflate}, + + /* inflate */ + {"inflateInit_nif", 3, zlib_inflateInit}, + {"inflateSetDictionary_nif", 2, zlib_inflateSetDictionary}, + {"inflateGetDictionary_nif", 1, zlib_inflateGetDictionary}, + {"inflateReset_nif", 1, zlib_inflateReset}, + {"inflateEnd_nif", 1, zlib_inflateEnd}, + {"inflate_nif", 4, zlib_inflate}, + + /* running checksum */ + {"crc32_nif", 1, zlib_crc32}, + + /* open & close */ + {"close_nif", 1, zlib_close}, + {"open_nif", 0, zlib_open}, + + /* The stash keeps a single term alive across calls, and is used in + * exception_on_need_dict/1 to retain the old error behavior, and for + * saving data flushed through deflateParams/3. */ + {"getStash_nif", 1, zlib_getStash}, + {"clearStash_nif", 1, zlib_clearStash}, + {"setStash_nif", 2, zlib_setStash}, + + /* DEPRECATED: buffer size for inflateChunk */ + {"getBufSize_nif", 1, zlib_getBufSize}, + {"setBufSize_nif", 2, zlib_setBufSize}, + + {"enqueue_nif", 2, zlib_enqueue_input}, +}; + +ERL_NIF_INIT(zlib, nif_funcs, load, NULL, upgrade, unload) + +static void gc_zlib(ErlNifEnv *env, void* data); + +static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) +{ + am_not_on_controlling_process = + enif_make_atom(env, "not_on_controlling_process"); + + am_not_initialized = enif_make_atom(env, "not_initialized"); + am_already_initialized = enif_make_atom(env, "already_initialized"); + + am_ok = enif_make_atom(env, "ok"); + am_error = enif_make_atom(env, "error"); + + am_continue = enif_make_atom(env, "continue"); + am_finished = enif_make_atom(env, "finished"); + + am_not_supported = enif_make_atom(env, "not_supported"); + am_need_dictionary = enif_make_atom(env, "need_dictionary"); + + am_empty = enif_make_atom(env, "empty"); + + am_stream_end = enif_make_atom(env, "stream_end"); + am_stream_error = enif_make_atom(env, "stream_error"); + am_data_error = enif_make_atom(env, "data_error"); + am_mem_error = enif_make_atom(env, "mem_error"); + am_buf_error = enif_make_atom(env, "buf_error"); + am_version_error = enif_make_atom(env, "version_error"); + am_unknown_error = enif_make_atom(env, "unknown_error"); + + rtype_zlib = enif_open_resource_type(env, NULL, + "gc_zlib", gc_zlib, ERL_NIF_RT_CREATE, NULL); + *priv_data = NULL; + + return 0; +} + +static void unload(ErlNifEnv *env, void* priv_data) +{ + +} + +static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + if(*old_priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(*priv_data != NULL) { + return -1; /* Don't know how to do that */ + } + if(load(env, priv_data, load_info)) { + return -1; + } + return 0; +} + +static void* zlib_alloc(void* data, unsigned int items, unsigned int size) +{ + return (void*) enif_alloc(items * size); +} + +static void zlib_free(void* data, void* addr) +{ + enif_free(addr); +} + +static ERL_NIF_TERM zlib_return(ErlNifEnv *env, int code) { + ERL_NIF_TERM reason; + switch(code) { + case Z_OK: + reason = am_ok; + break; + case Z_STREAM_END: + reason = am_stream_end; + break; + case Z_ERRNO: + reason = enif_make_int(env, errno); + break; + case Z_STREAM_ERROR: + reason = enif_raise_exception(env, am_stream_error); + break; + case Z_DATA_ERROR: + reason = enif_raise_exception(env, am_data_error); + break; + case Z_MEM_ERROR: + reason = am_mem_error; + break; + case Z_BUF_ERROR: + reason = am_buf_error; + break; + case Z_VERSION_ERROR: + reason = am_version_error; + break; + default: + reason = am_unknown_error; + break; + } + return reason; +} + +static void gc_zlib(ErlNifEnv *env, void* data) { + zlib_data_t *d = (zlib_data_t*)data; + + if(d->state == ST_DEFLATE) { + deflateEnd(&d->s); + } else if(d->state == ST_INFLATE) { + inflateEnd(&d->s); + } + + if(d->state != ST_CLOSED) { + enif_ioq_destroy(d->input_queue); + + if(d->stash_env != NULL) { + enif_free_env(d->stash_env); + } + + d->state = ST_CLOSED; + } +} + +static int get_zlib_data(ErlNifEnv *env, ERL_NIF_TERM opaque, zlib_data_t **d) { + return enif_get_resource(env, opaque, rtype_zlib, (void **)d); +} + +static int zlib_process_check(ErlNifEnv *env, zlib_data_t *d) { + ErlNifPid current_process; + + enif_self(env, ¤t_process); + + return enif_is_identical(enif_make_pid(env, ¤t_process), + enif_make_pid(env, &d->controlling_process)); +} + +static void zlib_reset_input(zlib_data_t *d) { + enif_ioq_destroy(d->input_queue); + d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + + if(d->stash_env != NULL) { + enif_free_env(d->stash_env); + d->stash_env = NULL; + d->stash_term = NIL; + } +} + +static int zlib_flush_queue(int (*codec)(z_stream*, int), ErlNifEnv *env, + zlib_data_t *d, size_t input_limit, ErlNifBinary *output_buffer, int flush, + size_t *bytes_produced, size_t *bytes_consumed, size_t *bytes_remaining) { + + int vec_len, vec_idx; + SysIOVec *input_vec; + int res; + + input_vec = enif_ioq_peek(d->input_queue, &vec_len); + vec_idx = 0; + res = Z_OK; + + *bytes_produced = 0; + *bytes_consumed = 0; + + d->s.avail_out = output_buffer->size; + d->s.next_out = output_buffer->data; + + while(res == Z_OK && vec_idx < vec_len && *bytes_consumed < input_limit) { + size_t timeslice_percent, block_consumed, block_size; + + block_size = MIN(input_vec[vec_idx].iov_len, input_limit); + + d->s.next_in = input_vec[vec_idx].iov_base; + d->s.avail_in = block_size; + + res = codec(&d->s, Z_NO_FLUSH); + + ASSERT(d->s.avail_in == 0 || d->s.avail_out == 0 || res != Z_OK); + + block_consumed = block_size - d->s.avail_in; + *bytes_consumed += block_consumed; + + if(d->want_input_crc) { + d->input_crc = + crc32(d->input_crc, input_vec[vec_idx].iov_base, block_consumed); + } + + timeslice_percent = (100 * block_consumed) / input_limit; + if(enif_consume_timeslice(env, MAX(1, timeslice_percent))) { + break; + } + + vec_idx++; + } + + if(!enif_ioq_deq(d->input_queue, *bytes_consumed, bytes_remaining)) { + *bytes_remaining = 0; + res = Z_BUF_ERROR; + } + + if(res == Z_OK && flush != Z_NO_FLUSH && (*bytes_remaining == 0)) { + d->s.next_in = NULL; + d->s.avail_in = 0; + + res = codec(&d->s, flush); + } + + *bytes_produced = output_buffer->size - d->s.avail_out; + + return res; +} + +static ERL_NIF_TERM zlib_codec(int (*codec)(z_stream*, int), + ErlNifEnv *env, zlib_data_t *d, + int input_chunk_size, + int output_chunk_size, + int flush) { + + size_t bytes_produced, bytes_consumed, bytes_remaining; + ErlNifBinary output_buffer; + int res; + + if(!enif_alloc_binary(output_chunk_size, &output_buffer)) { + return zlib_return(env, Z_MEM_ERROR); + } + + res = zlib_flush_queue(codec, env, d, input_chunk_size, &output_buffer, + flush, &bytes_produced, &bytes_consumed, &bytes_remaining); + + if(res < 0 && res != Z_BUF_ERROR) { + enif_release_binary(&output_buffer); + return zlib_return(env, res); + } + + if(res == Z_STREAM_END) { + d->eos_seen = 1; + } + + if(d->want_output_crc) { + d->output_crc = + crc32(d->output_crc, output_buffer.data, bytes_produced); + } + + if(bytes_consumed == 0 && bytes_produced == 0 && bytes_remaining != 0) { + /* Die if we've made zero progress; this should not happen on + * well-formed input. */ + + enif_release_binary(&output_buffer); + return zlib_return(env, Z_DATA_ERROR); + } else { + ERL_NIF_TERM flushed_output; + + if(bytes_produced > 0) { + if(bytes_produced < output_buffer.size) { + enif_realloc_binary(&output_buffer, bytes_produced); + } + + flushed_output = + enif_make_list1(env, enif_make_binary(env, &output_buffer)); + } else { + enif_release_binary(&output_buffer); + flushed_output = enif_make_list(env, 0); + } + + if(bytes_remaining == 0 && bytes_produced < output_chunk_size) { + return enif_make_tuple2(env, am_finished, flushed_output); + } else if(res != Z_NEED_DICT) { + return enif_make_tuple2(env, am_continue, flushed_output); + } + + return enif_make_tuple3(env, am_need_dictionary, + enif_make_int(env, d->s.adler), flushed_output); + } +} + +/* zlib nifs */ + +static ERL_NIF_TERM zlib_getStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(d->stash_env == NULL) { + return am_empty; + } + + return enif_make_tuple2(env, am_ok, enif_make_copy(env, d->stash_term)); +} + +static ERL_NIF_TERM zlib_clearStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->stash_env == NULL) { + return enif_raise_exception(env, am_error); + } + + enif_free_env(d->stash_env); + d->stash_env = NULL; + d->stash_term = NIL; + + return am_ok; +} + +static ERL_NIF_TERM zlib_setStash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->stash_env != NULL) { + return enif_raise_exception(env, am_error); + } + + d->stash_env = enif_alloc_env(); + d->stash_term = enif_make_copy(d->stash_env, argv[1]); + + return am_ok; +} + +static ERL_NIF_TERM zlib_open(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ERL_NIF_TERM result; + + d = (zlib_data_t *) enif_alloc_resource(rtype_zlib, sizeof(zlib_data_t)); + + memset(&d->s, 0, sizeof(z_stream)); + + enif_self(env, &d->controlling_process); + + d->input_queue = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + + d->s.zalloc = zlib_alloc; + d->s.zfree = zlib_free; + d->s.opaque = d; + d->s.data_type = Z_BINARY; + + d->eos_behavior = EOS_BEHAVIOR_CUT; + d->eos_seen = 0; + + d->state = ST_NONE; + + d->want_output_crc = 0; + d->want_input_crc = 0; + d->is_raw_stream = 0; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + + d->stash_env = NULL; + d->stash_term = NIL; + + d->inflateChunk_buffer_size = 4000; + + result = enif_make_resource(env, d); + enif_release_resource(d); + + return result; +} + +static ERL_NIF_TERM zlib_close(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + /* strictly speaking not needed since the gc will handle this */ + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state == ST_CLOSED) { + return enif_raise_exception(env, am_not_initialized); + } + + gc_zlib(env, d); + + return am_ok; +} + +/* deflate */ + +static ERL_NIF_TERM zlib_deflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int level, method, windowBits, memLevel, strategy, res; + + if(argc != 6 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &level) + || !enif_get_int(env, argv[2], &method) + || !enif_get_int(env, argv[3], &windowBits) + || !enif_get_int(env, argv[4], &memLevel) + || !enif_get_int(env, argv[5], &strategy)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_NONE) { + return enif_raise_exception(env, am_already_initialized); + } + + res = deflateInit2(&d->s, level, method, windowBits, memLevel, strategy); + + if(res == Z_OK) { + d->state = ST_DEFLATE; + d->eos_seen = 0; + + d->is_raw_stream = (windowBits < 0); + + d->want_output_crc = 0; + d->want_input_crc = d->is_raw_stream; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ErlNifBinary bin; + int res; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d) + || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if((res = deflateSetDictionary(&d->s, bin.data, bin.size)) == Z_OK) { + uLong checksum = d->s.adler; + + /* d->s.adler is not updated in raw deflate mode, so we'll calculate it + * ourselves in case the user wants to rely on that behavior. */ + if(d->is_raw_stream) { + checksum = adler32(0, bin.data, bin.size); + } + + return enif_make_int(env, checksum); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = deflateReset(&d->s); + + d->input_crc = crc32(0L, Z_NULL, 0); + d->eos_seen = 0; + + zlib_reset_input(d); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = deflateEnd(&d->s); + + if(res == Z_OK && enif_ioq_size(d->input_queue) > 0) { + res = Z_DATA_ERROR; + } + + zlib_reset_input(d); + d->state = ST_NONE; + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res, level, strategy; + + if(argc != 3 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &level) + || !enif_get_int(env, argv[2], &strategy)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + /* deflateParams will flush everything currently in the stream, corrupting + * the heap unless it's empty. We therefore pretend to have a full output + * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */ + d->s.avail_out = 0; + res = deflateParams(&d->s, level, strategy); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_deflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int input_chunk_size, output_chunk_size, flush; + + if(argc != 4 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &input_chunk_size) + || !enif_get_int(env, argv[2], &output_chunk_size) + || !enif_get_int(env, argv[3], &flush)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + return zlib_codec(&deflate, env, d, input_chunk_size, output_chunk_size, flush); +} + +/* inflate */ + +static ERL_NIF_TERM zlib_inflateInit(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int windowBits, eosBehavior, res; + + if(argc != 3 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &windowBits) + || !enif_get_int(env, argv[2], &eosBehavior)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_NONE) { + return enif_raise_exception(env, am_already_initialized); + } + + res = inflateInit2(&d->s, windowBits); + + if(res == Z_OK) { + d->state = ST_INFLATE; + + d->eos_behavior = eosBehavior; + d->eos_seen = 0; + + d->is_raw_stream = (windowBits < 0); + + d->want_output_crc = d->is_raw_stream; + d->want_input_crc = 0; + + d->output_crc = crc32(0L, Z_NULL, 0); + d->input_crc = crc32(0L, Z_NULL, 0); + } + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflateSetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + ErlNifBinary bin; + int res; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d) + || !enif_inspect_iolist_as_binary(env, argv[1], &bin)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateSetDictionary(&d->s, bin.data, bin.size); + + return zlib_return(env, res); +} + +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY +/* Work around broken build system with runtime version test */ +static int zlib_supports_inflateGetDictionary(void) { + static int supportsGetDictionary = -1; + +#if defined(__APPLE__) && defined(__MACH__) + if(supportsGetDictionary < 0) { + unsigned int v[4] = {0, 0, 0, 0}; + unsigned hexver; + + sscanf(zlibVersion(), "%u.%u.%u.%u", &v[0], &v[1], &v[2], &v[3]); + + hexver = (v[0] << (8*3)) | (v[1] << (8*2)) | (v[2] << (8)) | v[3]; + supportsGetDictionary = (hexver >= 0x1020701); /* 1.2.7.1 */ + } +#endif + + return supportsGetDictionary; +} +#endif + +static ERL_NIF_TERM zlib_inflateGetDictionary(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY + if(zlib_supports_inflateGetDictionary()) { + ErlNifBinary obin; + uInt len; + int res; + + enif_alloc_binary(INFL_DICT_SZ, &obin); + len = 0; + + if((res = inflateGetDictionary(&d->s, obin.data, &len)) < 0) { + enif_release_binary(&obin); + return zlib_return(env, res); + } + + enif_realloc_binary(&obin, (size_t)len); + return enif_make_binary(env, &obin); + } +#endif + + return enif_raise_exception(env, am_not_supported); +} + +static ERL_NIF_TERM zlib_inflateReset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateReset(&d->s); + + d->output_crc = crc32(0L, Z_NULL, 0); + d->eos_seen = 0; + + zlib_reset_input(d); + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + int res; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + res = inflateEnd(&d->s); + + if(res == Z_OK && (!d->eos_seen || enif_ioq_size(d->input_queue) > 0)) { + res = Z_DATA_ERROR; + } + + zlib_reset_input(d); + d->state = ST_NONE; + + return zlib_return(env, res); +} + +static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + int input_chunk_size, output_chunk_size, flush; + + if(argc != 4 || !get_zlib_data(env, argv[0], &d) + || !enif_get_int(env, argv[1], &input_chunk_size) + || !enif_get_int(env, argv[2], &output_chunk_size) + || !enif_get_int(env, argv[3], &flush)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if(d->eos_seen) { + int res; + + switch(d->eos_behavior) { + case EOS_BEHAVIOR_ERROR: + return zlib_return(env, Z_DATA_ERROR); + case EOS_BEHAVIOR_RESET: + res = inflateReset(&d->s); + + if(res != Z_OK) { + return zlib_return(env, res); + } + + d->eos_seen = 0; + break; + case EOS_BEHAVIOR_CUT: + zlib_reset_input(d); + + return enif_make_tuple2(env, am_finished, enif_make_list(env, 0)); + } + } + + return zlib_codec(&inflate, env, d, input_chunk_size, output_chunk_size, flush); +} + +static ERL_NIF_TERM zlib_crc32(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(d->state == ST_DEFLATE) { + return enif_make_ulong(env, d->input_crc); + } else if(d->state == ST_INFLATE) { + return enif_make_ulong(env, d->output_crc); + } + + return enif_raise_exception(env, am_not_initialized); +} + +static ERL_NIF_TERM zlib_getBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 1 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + return enif_make_int(env, d->inflateChunk_buffer_size); +} + +static ERL_NIF_TERM zlib_setBufSize(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } + + if(!enif_get_int(env, argv[1], &d->inflateChunk_buffer_size)) { + return enif_make_badarg(env); + } + + return am_ok; +} + +static ERL_NIF_TERM zlib_enqueue_input(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + zlib_data_t *d; + + ErlNifIOVec prealloc, *iovec = &prealloc; + ERL_NIF_TERM tail; + + if(argc != 2 || !get_zlib_data(env, argv[0], &d)) { + return enif_make_badarg(env); + } else if(!zlib_process_check(env, d)) { + return enif_raise_exception(env, am_not_on_controlling_process); + } else if(d->state != ST_DEFLATE && d->state != ST_INFLATE) { + return enif_raise_exception(env, am_not_initialized); + } + + if(!enif_inspect_iovec(env, 256, argv[1], &tail, &iovec)) { + return enif_make_badarg(env); + } else if(!enif_ioq_enqv(d->input_queue, iovec, 0)) { + return enif_make_badarg(env); + } + + if(!enif_is_empty_list(env, tail)) { + return enif_make_tuple2(env, am_continue, tail); + } + + return am_ok; +} diff --git a/erts/emulator/pcre/README.pcre_update.md b/erts/emulator/pcre/README.pcre_update.md index 8caf575d31..599e3d0d12 100644 --- a/erts/emulator/pcre/README.pcre_update.md +++ b/erts/emulator/pcre/README.pcre_update.md @@ -2,7 +2,7 @@ ## The basic changes to the PCRE library -To work with the Erlang VM, PCRE has been changed in two important ways: +To work with the Erlang VM, PCRE has been changed in three important ways: 1. The main execution machine in pcre\_exec has been modified so that matching can be interrupted and restarted. This functionality utilizes diff --git a/erts/emulator/pcre/local_config.h b/erts/emulator/pcre/local_config.h index e90f4dcada..c6af423d72 100644 --- a/erts/emulator/pcre/local_config.h +++ b/erts/emulator/pcre/local_config.h @@ -86,4 +86,4 @@ #define SUPPORT_UTF /* Version number of package */ -#define VERSION "8.40" +#define VERSION "8.41" diff --git a/erts/emulator/pcre/pcre-8.40.tar.bz2 b/erts/emulator/pcre/pcre-8.40.tar.bz2 Binary files differdeleted file mode 100644 index 6147917f4e..0000000000 --- a/erts/emulator/pcre/pcre-8.40.tar.bz2 +++ /dev/null diff --git a/erts/emulator/pcre/pcre-8.41.tar.bz2 b/erts/emulator/pcre/pcre-8.41.tar.bz2 Binary files differnew file mode 100644 index 0000000000..1798432dc9 --- /dev/null +++ b/erts/emulator/pcre/pcre-8.41.tar.bz2 diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 9cbd9c0293..ab8f40cfc1 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE. /* The current PCRE version information. */ #define PCRE_MAJOR 8 -#define PCRE_MINOR 40 +#define PCRE_MINOR 41 #define PCRE_PRERELEASE -#define PCRE_DATE 2017-01-11 +#define PCRE_DATE 2017-07-05 /* When an application links to a PCRE DLL in Windows, the symbols that are imported have to be identified as such. When building PCRE, the appropriate diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c index 6e841c9cf8..e79284ab79 100644 --- a/erts/emulator/pcre/pcre_compile.c +++ b/erts/emulator/pcre/pcre_compile.c @@ -5740,6 +5740,21 @@ for (;; ptr++) ptr = p - 1; /* Character before the next significant one. */ } + /* We also need to skip over (?# comments, which are not dependent on + extended mode. */ + + if (ptr[1] == CHAR_LEFT_PARENTHESIS && ptr[2] == CHAR_QUESTION_MARK && + ptr[3] == CHAR_NUMBER_SIGN) + { + ptr += 4; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) + { + *errorcodeptr = ERR18; + goto FAILED; + } + } + /* If the next character is '+', we have a possessive quantifier. This implies greediness, whatever the setting of the PCRE_UNGREEDY option. If the next character is '?' this is a minimizing repeat, by default, @@ -8211,7 +8226,6 @@ for (;; ptr++) if (mclength == 1 || req_caseopt == 0) { - firstchar = mcbuffer[0] | req_caseopt; firstchar = mcbuffer[0]; firstcharflags = req_caseopt; diff --git a/erts/emulator/pcre/pcre_dfa_exec.c b/erts/emulator/pcre/pcre_dfa_exec.c index 529f40685b..c859d67fc7 100644 --- a/erts/emulator/pcre/pcre_dfa_exec.c +++ b/erts/emulator/pcre/pcre_dfa_exec.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language (but see below for why this module is different). Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge + Copyright (c) 1997-2017 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -2626,7 +2626,7 @@ for (;;) if (isinclass) { int max = (int)GET2(ecode, 1 + IMM2_SIZE); - if (*ecode == OP_CRPOSRANGE) + if (*ecode == OP_CRPOSRANGE && count >= (int)GET2(ecode, 1)) { active_count--; /* Remove non-match possibility */ next_active_state--; diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 0f682d3daf..6708ba92a6 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -755,7 +755,7 @@ if (ecode == NULL) return match((PCRE_PUCHAR)&rdepth, NULL, NULL, 0, NULL, NULL, 1); else { - int len = (char *)&rdepth - (char *)eptr; + int len = (int)((char *)&rdepth - (char *)eptr); return (len > 0)? -len : len; } } diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index cc4f171438..c84dcb5a38 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2791,6 +2791,9 @@ extern const pcre_uint8 PRIV(ucd_stage1)[]; extern const pcre_uint16 PRIV(ucd_stage2)[]; extern const pcre_uint32 PRIV(ucp_gentype)[]; extern const pcre_uint32 PRIV(ucp_gbtable)[]; +#ifdef COMPILE_PCRE32 +extern const ucd_record PRIV(dummy_ucd_record)[]; +#endif #ifdef SUPPORT_JIT extern const int PRIV(ucp_typerange)[]; #endif @@ -2799,10 +2802,16 @@ extern const int PRIV(ucp_typerange)[]; /* UCD access macros */ #define UCD_BLOCK_SIZE 128 -#define GET_UCD(ch) (PRIV(ucd_records) + \ +#define REAL_GET_UCD(ch) (PRIV(ucd_records) + \ PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \ UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE]) +#ifdef COMPILE_PCRE32 +#define GET_UCD(ch) ((ch > 0x10ffff)? PRIV(dummy_ucd_record) : REAL_GET_UCD(ch)) +#else +#define GET_UCD(ch) REAL_GET_UCD(ch) +#endif + #define UCD_CHARTYPE(ch) GET_UCD(ch)->chartype #define UCD_SCRIPT(ch) GET_UCD(ch)->script #define UCD_CATEGORY(ch) PRIV(ucp_gentype)[UCD_CHARTYPE(ch)] diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c index 89400498f0..932ca2c389 100644 --- a/erts/emulator/pcre/pcre_jit_compile.c +++ b/erts/emulator/pcre/pcre_jit_compile.c @@ -487,7 +487,7 @@ typedef struct compare_context { #undef CMP /* Used for accessing the elements of the stack. */ -#define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_sw)) +#define STACK(i) ((i) * (int)sizeof(sljit_sw)) #define TMP1 SLJIT_R0 #define TMP2 SLJIT_R2 @@ -552,13 +552,15 @@ the start pointers when the end of the capturing group has not yet reached. */ sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)) #define CMPTO(type, src1, src1w, src2, src2w, label) \ sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label)) -#define OP_FLAGS(op, dst, dstw, src, srcw, type) \ - sljit_emit_op_flags(compiler, (op), (dst), (dstw), (src), (srcw), (type)) +#define OP_FLAGS(op, dst, dstw, type) \ + sljit_emit_op_flags(compiler, (op), (dst), (dstw), (type)) #define GET_LOCAL_BASE(dst, dstw, offset) \ sljit_get_local_base(compiler, (dst), (dstw), (offset)) #define READ_CHAR_MAX 0x7fffffff +#define INVALID_UTF_CHAR 888 + static pcre_uchar *bracketend(pcre_uchar *cc) { SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); @@ -784,7 +786,7 @@ switch(*cc) default: /* All opcodes are supported now! */ - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return NULL; } } @@ -1660,9 +1662,9 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setsom_found = TRUE; } cc += 1; @@ -1676,9 +1678,9 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setmark_found = TRUE; } cc += 1 + 2 + cc[1]; @@ -1689,27 +1691,27 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setsom_found = TRUE; } if (common->mark_ptr != 0 && !setmark_found) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); setmark_found = TRUE; } if (common->capture_last_ptr != 0 && !capture_last_found) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); capture_last_found = TRUE; } cc += 1 + LINK_SIZE; @@ -1723,20 +1725,20 @@ while (cc < ccend) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); capture_last_found = TRUE; } offset = (GET2(cc, 1 + LINK_SIZE)) << 1; OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset)); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0); - stackpos += (int)sizeof(sljit_sw); + stackpos -= (int)sizeof(sljit_sw); cc += 1 + LINK_SIZE + IMM2_SIZE; break; @@ -1887,18 +1889,17 @@ BOOL tmp1empty = TRUE; BOOL tmp2empty = TRUE; pcre_uchar *alternative; enum { - start, loop, end } status; -status = save ? start : loop; -stackptr = STACK(stackptr - 2); +status = loop; +stackptr = STACK(stackptr); stacktop = STACK(stacktop - 1); if (!save) { - stackptr += (needs_control_head ? 2 : 1) * sizeof(sljit_sw); + stacktop -= (needs_control_head ? 2 : 1) * sizeof(sljit_sw); if (stackptr < stacktop) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); @@ -1914,196 +1915,186 @@ if (!save) /* The tmp1next must be TRUE in either way. */ } +SLJIT_ASSERT(common->recursive_head_ptr != 0); + do { count = 0; - switch(status) + if (cc >= ccend) { - case start: - SLJIT_ASSERT(save && common->recursive_head_ptr != 0); + if (!save) + break; + count = 1; srcw[0] = common->recursive_head_ptr; if (needs_control_head) { SLJIT_ASSERT(common->control_head_ptr != 0); count = 2; - srcw[1] = common->control_head_ptr; + srcw[0] = common->control_head_ptr; + srcw[1] = common->recursive_head_ptr; + } + status = end; + } + else switch(*cc) + { + case OP_KET: + if (PRIVATE_DATA(cc) != 0) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); + cc += PRIVATE_DATA(cc + 1); } - status = loop; + cc += 1 + LINK_SIZE; + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCOND: + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(srcw[0] != 0); + cc += 1 + LINK_SIZE; break; - case loop: - if (cc >= ccend) + case OP_CBRA: + case OP_SCBRA: + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) { - status = end; - break; + count = 1; + srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); } + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; - switch(*cc) - { - case OP_KET: - if (PRIVATE_DATA(cc) != 0) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); - cc += PRIVATE_DATA(cc + 1); - } - cc += 1 + LINK_SIZE; - break; + case OP_CBRAPOS: + case OP_SCBRAPOS: + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); + SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCOND: + case OP_COND: + /* Might be a hidden SCOND. */ + alternative = cc + GET(cc, 1); + if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) + { count = 1; srcw[0] = PRIVATE_DATA(cc); SLJIT_ASSERT(srcw[0] != 0); - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_SCBRA: - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - { - count = 1; - srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - } - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; + } + cc += 1 + LINK_SIZE; + break; - case OP_CBRAPOS: - case OP_SCBRAPOS: - count = 2; + CASE_ITERATOR_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; srcw[0] = PRIVATE_DATA(cc); - srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - /* Might be a hidden SCOND. */ - alternative = cc + GET(cc, 1); - if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(srcw[0] != 0); - } - cc += 1 + LINK_SIZE; - break; - - CASE_ITERATOR_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - } - cc += 2; + } + cc += 2; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2; + CASE_ITERATOR_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2 + IMM2_SIZE; + CASE_ITERATOR_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2 + IMM2_SIZE; #ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) + CASE_ITERATOR_TYPE_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1 + IMM2_SIZE; + break; + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); +#else + size = 1 + 32 / (int)sizeof(pcre_uchar); +#endif + if (PRIVATE_DATA(cc)) + switch(get_class_iterator_size(cc + size)) { + case 1: count = 1; srcw[0] = PRIVATE_DATA(cc); - } - cc += 1; - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { + case 2: count = 2; srcw[0] = PRIVATE_DATA(cc); srcw[1] = srcw[0] + sizeof(sljit_sw); - } - cc += 1; - break; + break; - CASE_ITERATOR_TYPE_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); + default: + SLJIT_UNREACHABLE(); + break; } - cc += 1 + IMM2_SIZE; - break; - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); -#else - size = 1 + 32 / (int)sizeof(pcre_uchar); -#endif - if (PRIVATE_DATA(cc)) - switch(get_class_iterator_size(cc + size)) - { - case 1: - count = 1; - srcw[0] = PRIVATE_DATA(cc); - break; - - case 2: - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - cc += size; - break; - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } + cc += size; break; - case end: - SLJIT_ASSERT_STOP(); + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); break; } @@ -2312,7 +2303,7 @@ static SLJIT_INLINE void count_match(compiler_common *common) { DEFINE_COMPILER; -OP2(SLJIT_SUB | SLJIT_SET_E, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); +OP2(SLJIT_SUB | SLJIT_SET_Z, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO)); } @@ -2322,7 +2313,7 @@ static SLJIT_INLINE void allocate_stack(compiler_common *common, int size) DEFINE_COMPILER; SLJIT_ASSERT(size > 0); -OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); #ifdef DESTROY_REGISTERS OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345); OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); @@ -2330,7 +2321,7 @@ OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); #endif -add_stub(common, CMP(SLJIT_GREATER, STACK_TOP, 0, STACK_LIMIT, 0)); +add_stub(common, CMP(SLJIT_LESS, STACK_TOP, 0, STACK_LIMIT, 0)); } static SLJIT_INLINE void free_stack(compiler_common *common, int size) @@ -2338,7 +2329,7 @@ static SLJIT_INLINE void free_stack(compiler_common *common, int size) DEFINE_COMPILER; SLJIT_ASSERT(size > 0); -OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); } static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size) @@ -2396,7 +2387,7 @@ else OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1); loop = LABEL(); OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); } } @@ -2434,7 +2425,7 @@ else OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2); loop = LABEL(); OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); } @@ -2452,22 +2443,22 @@ static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *s { while (current != NULL) { - switch (current[-2]) + switch (current[1]) { case type_then_trap: break; case type_mark: - if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[-3]) == 0) - return current[-4]; + if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[2]) == 0) + return current[3]; break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } - SLJIT_ASSERT(current > (sljit_sw*)current[-1]); - current = (sljit_sw*)current[-1]; + SLJIT_ASSERT(current[0] == 0 || current < (sljit_sw*)current[0]); + current = (sljit_sw*)current[0]; } return -1; } @@ -2501,7 +2492,7 @@ OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw)); OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, loop); JUMPHERE(early_quit); @@ -3106,8 +3097,8 @@ if (common->utf) OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); /* Skip low surrogate if necessary. */ OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); return; @@ -3126,6 +3117,7 @@ struct sljit_jump *jump; if (nltype == NLTYPE_ANY) { add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO)); } else if (nltype == NLTYPE_ANYCRLF) @@ -3167,7 +3159,7 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); /* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -3181,7 +3173,7 @@ OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); jump = JUMP(SLJIT_NOT_ZERO); /* Three byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); @@ -3215,15 +3207,15 @@ OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); /* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_NOT_ZERO); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_NOT_ZERO); /* This code runs only in 8 bit mode. No need to shift the value. */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); @@ -3246,7 +3238,7 @@ struct sljit_jump *compare; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); @@ -3287,10 +3279,30 @@ static void do_getucd(compiler_common *common) /* Search the UCD record for the character comes in TMP1. Returns chartype in TMP1 and UCD offset in TMP2. */ DEFINE_COMPILER; +#ifdef COMPILE_PCRE32 +struct sljit_jump *jump; +#endif + +#if defined SLJIT_DEBUG && SLJIT_DEBUG +/* dummy_ucd_record */ +const ucd_record *record = GET_UCD(INVALID_UTF_CHAR); +SLJIT_ASSERT(record->script == ucp_Common && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); +SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0); +#endif SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8); sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +#ifdef COMPILE_PCRE32 +if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); @@ -3365,8 +3377,8 @@ if (newlinecheck) OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -3403,8 +3415,8 @@ if (common->utf) { singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); JUMPHERE(singlechar); @@ -3853,7 +3865,7 @@ while (TRUE) } } -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) static sljit_s32 character_to_int32(pcre_uchar chr) { @@ -4019,6 +4031,7 @@ instruction[0] = 0x0f; instruction[1] = 0xbc; instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); nomatch = JUMP(SLJIT_ZERO); @@ -4119,6 +4132,7 @@ instruction[0] = 0x0f; instruction[1] = 0xbc; instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); JUMPTO(SLJIT_ZERO, start); @@ -4155,18 +4169,8 @@ if (has_match_end) OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1)); -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) - if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); - } -#endif - { - quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP3, 0); - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - JUMPHERE(quit); - } + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); + sljit_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); } #if defined SUPPORT_UTF && !defined COMPILE_PCRE32 @@ -4174,11 +4178,11 @@ if (common->utf && offset > 0) utf_start = LABEL(); #endif -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) /* SSE2 accelerated first character search. */ -if (sljit_x86_is_sse2_available()) +if (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) { fast_forward_first_char2_sse2(common, char1, char2); @@ -4213,16 +4217,16 @@ if (sljit_x86_is_sse2_available()) if (offset > 0) OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); } - else if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - } else { - quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_PTR, 0, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - JUMPHERE(quit); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + if (has_match_end) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, TMP1, 0); + } + else + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, STR_END, 0); } if (has_match_end) @@ -4249,10 +4253,10 @@ else } else { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); found = JUMP(SLJIT_NOT_ZERO); } } @@ -4571,8 +4575,8 @@ if (common->nltype == NLTYPE_FIXED && common->newline > 255) firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -4616,8 +4620,8 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) JUMPHERE(foundcr); notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); #if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif @@ -4670,7 +4674,7 @@ if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE, OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); found = JUMP(SLJIT_NOT_ZERO); } @@ -4692,8 +4696,8 @@ if (common->utf) { CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); } @@ -4780,31 +4784,31 @@ struct sljit_jump *jump; struct sljit_label *mainloop; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0); -GET_LOCAL_BASE(TMP3, 0, 0); +OP1(SLJIT_MOV, TMP3, 0, STACK_TOP, 0); +GET_LOCAL_BASE(TMP1, 0, 0); /* Drop frames until we reach STACK_TOP. */ mainloop = LABEL(); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0); -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0); -jump = JUMP(SLJIT_SIG_LESS_EQUAL); - -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw)); +jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0); + +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -3 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); JUMPTO(SLJIT_JUMP, mainloop); JUMPHERE(jump); -jump = JUMP(SLJIT_SIG_LESS); -/* End of dropping frames. */ +jump = CMP(SLJIT_NOT_ZERO /* SIG_LESS */, TMP2, 0, SLJIT_IMM, 0); +/* End of reverting values. */ +OP1(SLJIT_MOV, STACK_TOP, 0, TMP3, 0); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); JUMPTO(SLJIT_JUMP, mainloop); } @@ -4837,11 +4841,11 @@ if (common->use_ucp) jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); JUMPHERE(jump); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); } @@ -4881,11 +4885,11 @@ if (common->use_ucp) jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); JUMPHERE(jump); } else @@ -4913,7 +4917,7 @@ else } set_jumps(skipread_list, LABEL()); -OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); +OP2(SLJIT_XOR | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); } @@ -5064,7 +5068,7 @@ switch(length) return TRUE; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return FALSE; } } @@ -5077,22 +5081,22 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5103,34 +5107,34 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); -OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); +OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5143,22 +5147,22 @@ DEFINE_COMPILER; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); #if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 #ifdef COMPILE_PCRE8 if (common->utf) { #endif - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); #ifdef COMPILE_PCRE8 } #endif #endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -5183,7 +5187,7 @@ label = LABEL(); OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); JUMPTO(SLJIT_NOT_ZERO, label); JUMPHERE(jump); @@ -5227,7 +5231,7 @@ OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0); JUMPHERE(jump); #endif jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); JUMPTO(SLJIT_NOT_ZERO, label); JUMPHERE(jump); @@ -5394,7 +5398,7 @@ do #endif default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } context->ucharptr = 0; @@ -5568,7 +5572,7 @@ while (*cc != XCL_END) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } cc += 2; @@ -5592,7 +5596,7 @@ if ((cc[-1] & XCL_HASPROP) == 0) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO)); } @@ -5625,7 +5629,7 @@ else if ((cc[-1] & XCL_MAP) != 0) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO)); #ifdef COMPILE_PCRE8 @@ -5644,6 +5648,15 @@ if (needstype || needsscript) if (needschar && !charsaved) OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); +#ifdef COMPILE_PCRE32 + if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); @@ -5735,14 +5748,14 @@ while (*cc != XCL_END) if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_EQUAL); numberofcmps++; } else if (numberofcmps > 0) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); numberofcmps = 0; } @@ -5761,14 +5774,14 @@ while (*cc != XCL_END) if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); numberofcmps++; } else if (numberofcmps > 0) { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); numberofcmps = 0; } @@ -5793,12 +5806,12 @@ while (*cc != XCL_END) break; case PT_LAMP: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; @@ -5820,33 +5833,33 @@ while (*cc != XCL_END) case PT_SPACE: case PT_PXSPACE: SET_CHAR_OFFSET(9); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); SET_TYPE_OFFSET(ucp_Zl); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_WORD: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); /* Fall through. */ case PT_ALNUM: SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); SET_TYPE_OFFSET(ucp_Nd); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; @@ -5868,8 +5881,8 @@ while (*cc != XCL_END) OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); other_cases += 2; } else if (is_powerof2(other_cases[2] ^ other_cases[1])) @@ -5881,63 +5894,63 @@ while (*cc != XCL_END) OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); - OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); + OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); other_cases += 3; } else { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); } while (*other_cases != NOTACHAR) { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); } jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_UCNC: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); SET_CHAR_OFFSET(0xa0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_GREATER_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_GREATER_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; case PT_PXGRAPH: /* C and Z groups are the farthest two groups. */ SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); /* In case of ucp_Cf, we overwrite the result. */ SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); JUMPHERE(jump); jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); @@ -5946,21 +5959,21 @@ while (*cc != XCL_END) case PT_PXPRINT: /* C and Z groups are the farthest two groups. */ SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_NOT_EQUAL); jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); /* In case of ucp_Cf, we overwrite the result. */ SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); JUMPHERE(jump); jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); @@ -5968,21 +5981,21 @@ while (*cc != XCL_END) case PT_PXPUNCT: SET_TYPE_OFFSET(ucp_Sc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_LESS_EQUAL); SET_TYPE_OFFSET(ucp_Pc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } cc += 2; @@ -6028,6 +6041,7 @@ switch(type) case OP_NOT_WORD_BOUNDARY: case OP_WORD_BOUNDARY: add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6043,10 +6057,10 @@ switch(type) else { jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_NOT_EQUAL); add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL)); check_partial(common, TRUE); add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); @@ -6068,9 +6082,9 @@ switch(type) OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); jump[2] = JUMP(SLJIT_GREATER); - add_jump(compiler, backtracks, JUMP(SLJIT_LESS)); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL) /* LESS */); /* Equal. */ OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); @@ -6089,6 +6103,7 @@ switch(type) read_char_range(common, common->nlmin, common->nlmax, TRUE); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0)); add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); } @@ -6204,7 +6219,7 @@ switch(type) label = LABEL(); add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0)); skip_char_back(common); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -6217,7 +6232,7 @@ switch(type) check_start_used_ptr(common); return cc + LINK_SIZE; } -SLJIT_ASSERT_STOP(); +SLJIT_UNREACHABLE(); return cc; } @@ -6250,7 +6265,7 @@ switch(type) #endif read_char8_type(common, type == OP_NOT_DIGIT); /* Flip the starting bit in the negative case. */ - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6264,7 +6279,7 @@ switch(type) else #endif read_char8_type(common, type == OP_NOT_WHITESPACE); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6278,7 +6293,7 @@ switch(type) else #endif read_char8_type(common, type == OP_NOT_WORDCHAR); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -6320,8 +6335,8 @@ switch(type) #elif defined COMPILE_PCRE16 jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); #endif @@ -6383,6 +6398,7 @@ switch(type) detect_partial_match(common, backtracks); read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE); add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6392,6 +6408,7 @@ switch(type) detect_partial_match(common, backtracks); read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE); add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -6418,7 +6435,7 @@ switch(type) OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable)); OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); JUMPTO(SLJIT_NOT_ZERO, label); OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); @@ -6587,7 +6604,7 @@ switch(type) OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 @@ -6604,7 +6621,7 @@ switch(type) return cc + GET(cc, 0) - 1; #endif } -SLJIT_ASSERT_STOP(); +SLJIT_UNREACHABLE(); return cc; } @@ -6790,9 +6807,9 @@ else #endif /* SUPPORT_UTF && SUPPORT_UCP */ { if (ref) - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); else - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); if (withchecks) jump = JUMP(SLJIT_ZERO); @@ -6883,7 +6900,7 @@ switch(type) cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE; break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -6897,7 +6914,7 @@ if (!minimize) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); /* Temporary release of STR_PTR. */ - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); /* Handles both invalid and empty cases. Since the minimum repeat, is zero the invalid case is basically the same as an empty case. */ if (ref) @@ -6910,7 +6927,7 @@ if (!minimize) zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); } /* Restore if not zero length. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); } else { @@ -7157,7 +7174,7 @@ return (*PUBL(callout))(callout_block); (((int)sizeof(PUBL(callout_block)) + 7) & ~7) #define CALLOUT_ARG_OFFSET(arg) \ - (-CALLOUT_ARG_SIZE + SLJIT_OFFSETOF(PUBL(callout_block), arg)) + SLJIT_OFFSETOF(PUBL(callout_block), arg) static SLJIT_INLINE pcre_uchar *compile_callout_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) { @@ -7187,7 +7204,8 @@ OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_pt /* Needed to save important temporary registers. */ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); -OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE); +/* SLJIT_R0 = arguments */ +OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0); GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START); sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout)); OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0); @@ -7195,12 +7213,12 @@ OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); /* Check return value. */ -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER)); if (common->forced_quit_label == NULL) - add_jump(compiler, &common->forced_quit, JUMP(SLJIT_SIG_LESS)); + add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */); else - JUMPTO(SLJIT_SIG_LESS, common->forced_quit_label); + JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->forced_quit_label); return cc + 2 + 2 * LINK_SIZE; } @@ -7321,7 +7339,7 @@ else allocate_stack(common, framesize + extrasize); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); if (needs_control_head) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); @@ -7392,22 +7410,22 @@ while (1) free_stack(common, extrasize); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } else { if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional) { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } else { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 2)); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); } } @@ -7418,25 +7436,25 @@ while (1) if (conditional) { if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? STACK(-2) : STACK(-1)); } else if (bra == OP_BRAZERO) { if (framesize < 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); else { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - extrasize)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); } - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); } else if (framesize >= 0) { /* For OP_BRA and OP_BRAMINZERO. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); } } add_jump(compiler, found, JUMP(SLJIT_JUMP)); @@ -7480,12 +7498,12 @@ if (common->positive_assert_quit != NULL) set_jumps(common->positive_assert_quit, LABEL()); SLJIT_ASSERT(framesize != no_stack); if (framesize < 0) - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); else { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); } JUMPHERE(jump); } @@ -7534,18 +7552,18 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) { /* We know that STR_PTR was stored on the top of the stack. */ if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); /* Keep the STR_PTR on the top of the stack. */ if (bra == OP_BRAZERO) { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); if (extrasize == 2) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); } else if (bra == OP_BRAMINZERO) { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); } } @@ -7554,13 +7572,13 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) if (bra == OP_BRA) { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 2) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize + 1)); } else { /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); if (extrasize == 2) { OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); @@ -7588,7 +7606,7 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); } set_jumps(backtrack->common.topbacktracks, LABEL()); } @@ -7675,23 +7693,23 @@ if (framesize < 0) } if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? sizeof(sljit_sw) : 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? STACK(-2) : STACK(-1)); /* TMP2 which is set here used by OP_KETRMAX below. */ if (ket == OP_KETRMAX) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); else if (ket == OP_KETRMIN) { /* Move the STR_PTR to the private_data_ptr. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); } } else { stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1; - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); if (ket == OP_KETRMAX) { @@ -7927,7 +7945,7 @@ if (bra == OP_BRAMINZERO) { /* Except when the whole stack frame must be saved. */ OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw)); + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2)); } JUMPHERE(skip); } @@ -8000,7 +8018,7 @@ if (opcode == OP_ONCE) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); } else if (ket == OP_KETRMAX || has_alternatives) @@ -8018,7 +8036,7 @@ if (opcode == OP_ONCE) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); stacksize = needs_control_head ? 1 : 0; if (ket != OP_KET || has_alternatives) @@ -8090,13 +8108,13 @@ if (opcode == OP_COND || opcode == OP_SCOND) slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); slot += common->name_entry_size; i--; while (i-- > 0) { OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); - OP2(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, STR_PTR, 0); + OP2(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, STR_PTR, 0); slot += common->name_entry_size; } OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); @@ -8111,7 +8129,7 @@ if (opcode == OP_COND || opcode == OP_SCOND) if (*matchingpath == OP_FAIL) stacksize = 0; - if (*matchingpath == OP_RREF) + else if (*matchingpath == OP_RREF) { stacksize = GET2(matchingpath, 1); if (common->currententry == NULL) @@ -8244,7 +8262,7 @@ if (ket == OP_KETRMAX) { if (has_alternatives) BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, rmax_label); /* Drop STR_PTR for greedy plus quantifier. */ if (opcode != OP_ONCE) @@ -8274,7 +8292,7 @@ if (ket == OP_KETRMAX) if (repeat_type == OP_EXACT) { count_match(common); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, rmax_label); } else if (repeat_type == OP_UPTO) @@ -8374,7 +8392,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -8452,7 +8470,7 @@ else OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); if (needs_control_head) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); stack = 0; if (!zero) @@ -8524,7 +8542,7 @@ while (*cc != OP_KETRPOS) { if (offset != 0) { - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); @@ -8535,10 +8553,10 @@ while (*cc != OP_KETRPOS) else { OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); if (opcode == OP_SBRAPOS) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw), STR_PTR, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), STACK(-framesize - 2), STR_PTR, 0); } /* Even if the match is empty, we need to reset the control head. */ @@ -8584,7 +8602,7 @@ while (*cc != OP_KETRPOS) else { OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); } } @@ -8601,7 +8619,7 @@ if (!zero) if (framesize < 0) add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0)); else /* TMP2 is set to [private_data_ptr] above. */ - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0)); + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), STACK(-stacksize), SLJIT_IMM, 0)); } /* None of them matched. */ @@ -8824,7 +8842,7 @@ if (exact > 1) OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); label = LABEL(); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -8832,7 +8850,7 @@ if (exact > 1) OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); label = LABEL(); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } } @@ -8862,7 +8880,7 @@ switch(opcode) if (opcode == OP_UPTO) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); jump = JUMP(SLJIT_ZERO); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); } @@ -8924,7 +8942,7 @@ switch(opcode) label = LABEL(); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); } compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); @@ -8944,7 +8962,7 @@ switch(opcode) OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); } @@ -8971,7 +8989,7 @@ switch(opcode) if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -9000,7 +9018,7 @@ switch(opcode) if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -9026,7 +9044,7 @@ switch(opcode) compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); if (opcode == OP_UPTO) { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); } @@ -9113,7 +9131,7 @@ switch(opcode) label = LABEL(); compile_char1_matchingpath(common, type, cc, &no_match, TRUE); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); set_jumps(no_match, LABEL()); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); @@ -9124,7 +9142,7 @@ switch(opcode) label = LABEL(); detect_partial_match(common, &no_match); compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); set_jumps(no_char1_match, LABEL()); @@ -9142,7 +9160,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -9264,7 +9282,7 @@ size = 3 + (size < 0 ? 0 : size); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); allocate_stack(common, size); if (size > 3) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); else OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start); @@ -9569,7 +9587,7 @@ while (cc < ccend) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); return; } if (cc == NULL) @@ -9677,7 +9695,7 @@ switch(opcode) case OP_MINUPTO: OP1(SLJIT_MOV, TMP1, 0, base, offset1); OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO)); OP1(SLJIT_MOV, base, offset1, TMP1, 0); @@ -9723,7 +9741,7 @@ switch(opcode) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } @@ -9831,7 +9849,7 @@ if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(assert_backtrack)->framesize - 1)); set_jumps(current->topbacktracks, LABEL()); } @@ -9841,7 +9859,7 @@ else if (bra == OP_BRAZERO) { /* We know there is enough place on the stack. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_backtrack)->matchingpath); JUMPHERE(brajump); @@ -9954,7 +9972,7 @@ else if (ket == OP_KETRMIN) else { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 2), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); } /* Drop STR_PTR for non-greedy plus quantifier. */ if (opcode != OP_ONCE) @@ -10060,7 +10078,7 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); } cond = JUMP(SLJIT_JUMP); set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL()); @@ -10201,7 +10219,7 @@ if (has_alternatives) { OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); } JUMPHERE(cond); } @@ -10256,7 +10274,7 @@ else if (opcode == OP_ONCE) JUMPHERE(once); /* Restore previous private_data_ptr */ if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 1)); else if (ket == OP_KETRMIN) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); @@ -10346,7 +10364,7 @@ if (current->topbacktracks) free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); JUMPHERE(jump); } -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracketpos_backtrack)->framesize - 1)); } static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current) @@ -10392,10 +10410,10 @@ if (opcode == OP_THEN || opcode == OP_THEN_ARG) jump = JUMP(SLJIT_JUMP); loop = LABEL(); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), -(int)sizeof(sljit_sw)); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); JUMPHERE(jump); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0, loop); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0, loop); add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP)); return; } @@ -10645,7 +10663,7 @@ while (current) break; default: - SLJIT_ASSERT_STOP(); + SLJIT_UNREACHABLE(); break; } current = current->prev; @@ -10684,7 +10702,7 @@ sljit_emit_fast_enter(compiler, TMP2, 0); count_match(common); allocate_stack(common, private_data_size + framesize + alternativesize); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0); -copy_private_data(common, ccbegin, ccend, TRUE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); +copy_private_data(common, ccbegin, ccend, TRUE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); if (needs_control_head) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0); @@ -10737,9 +10755,9 @@ if (common->quit != NULL) OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); if (needs_frame) { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); } OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); common->quit = NULL; @@ -10750,32 +10768,32 @@ set_jumps(common->accept, LABEL()); OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); if (needs_frame) { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); } OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1); JUMPHERE(jump); if (common->quit != NULL) set_jumps(common->quit, LABEL()); -copy_private_data(common, ccbegin, ccend, FALSE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); +copy_private_data(common, ccbegin, ccend, FALSE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); free_stack(common, private_data_size + framesize + alternativesize); if (needs_control_head) { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 2 * sizeof(sljit_sw)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-3)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0); OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); } else { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0); } -sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0); +sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), STACK(-1)); } #undef COMPILE_BACKTRACKINGPATH @@ -11237,7 +11255,7 @@ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0); -OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); +OP2(SLJIT_SUB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize)); jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); @@ -11391,10 +11409,10 @@ union { sljit_u8 local_space[MACHINE_STACK_SIZE]; struct sljit_stack local_stack; -local_stack.top = (sljit_sw)&local_space; -local_stack.base = local_stack.top; -local_stack.limit = local_stack.base + MACHINE_STACK_SIZE; -local_stack.max_limit = local_stack.limit; +local_stack.max_limit = local_space; +local_stack.limit = local_space; +local_stack.base = local_space + MACHINE_STACK_SIZE; +local_stack.top = local_space + MACHINE_STACK_SIZE; arguments->stack = &local_stack; convert_executable_func.executable_func = executable_func; return convert_executable_func.call_executable_func(arguments); diff --git a/erts/emulator/pcre/pcre_tables.c b/erts/emulator/pcre/pcre_tables.c index 2f6302e2e1..08e31f1460 100644 --- a/erts/emulator/pcre/pcre_tables.c +++ b/erts/emulator/pcre/pcre_tables.c @@ -6,7 +6,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge + Copyright (c) 1997-2017 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -162,7 +162,7 @@ const pcre_uint32 PRIV(ucp_gbtable[]) = { (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark), /* 5 SpacingMark */ (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbL)| /* 6 L */ - (1<<ucp_gbL)|(1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT), + (1<<ucp_gbV)|(1<<ucp_gbLV)|(1<<ucp_gbLVT), (1<<ucp_gbExtend)|(1<<ucp_gbSpacingMark)|(1<<ucp_gbV)| /* 7 V */ (1<<ucp_gbT), diff --git a/erts/emulator/pcre/pcre_ucd.c b/erts/emulator/pcre/pcre_ucd.c index 9b700c0785..2dd4b05751 100644 --- a/erts/emulator/pcre/pcre_ucd.c +++ b/erts/emulator/pcre/pcre_ucd.c @@ -38,6 +38,20 @@ const pcre_uint16 PRIV(ucd_stage2)[] = {0}; const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0}; #else +/* If the 32-bit library is run in non-32-bit mode, character values +greater than 0x10ffff may be encountered. For these we set up a +special record. */ + +#ifdef COMPILE_PCRE32 +const ucd_record PRIV(dummy_ucd_record)[] = {{ + ucp_Common, /* script */ + ucp_Cn, /* type unassigned */ + ucp_gbOther, /* grapheme break property */ + 0, /* case set */ + 0, /* other case */ + }}; +#endif + /* When recompiling tables with a new Unicode version, please check the types in this structure definition from pcre_internal.h (the actual field names will be different): diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index ad580e7d52..f93d4c1557 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -29,7 +29,6 @@ #endif #define ERL_CHECK_IO_C__ -#define ERTS_WANT_BREAK_HANDLING #ifndef WANT_NONBLOCKING # define WANT_NONBLOCKING #endif @@ -44,76 +43,79 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS -# include "safe_hash.h" -# define DRV_EV_STATE_HTAB_SIZE 1024 +#if 0 +#define DEBUG_PRINT(FMT, ...) erts_printf(FMT "\r\n", ##__VA_ARGS__) +#define DEBUG_PRINT_FD(FMT, STATE, ...) \ + DEBUG_PRINT("%d: " FMT " (ev=%s, ac=%s, flg=%d)", \ + (STATE) ? (STATE)->fd : (ErtsSysFdType)-1, ##__VA_ARGS__, \ + ev2str((STATE) ? (STATE)->events : ERTS_POLL_EV_NONE), \ + ev2str((STATE) ? (STATE)->active_events : ERTS_POLL_EV_NONE), \ + (STATE) ? (STATE)->flags : ERTS_EV_FLAG_CLEAR) +#define DEBUG_PRINT_MODE +#else +#define DEBUG_PRINT(...) #endif -typedef char EventStateType; -#define ERTS_EV_TYPE_NONE ((EventStateType) 0) -#define ERTS_EV_TYPE_DRV_SEL ((EventStateType) 1) /* driver_select */ -#define ERTS_EV_TYPE_DRV_EV ((EventStateType) 2) /* driver_event */ -#define ERTS_EV_TYPE_STOP_USE ((EventStateType) 3) /* pending stop_select */ -#define ERTS_EV_TYPE_NIF ((EventStateType) 4) /* enif_select */ -#define ERTS_EV_TYPE_STOP_NIF ((EventStateType) 5) /* pending nif stop */ - -typedef char EventStateFlags; -#define ERTS_EV_FLAG_USED ((EventStateFlags) 1) /* ERL_DRV_USE has been turned on */ -#define ERTS_EV_FLAG_DEFER_IN_EV ((EventStateFlags) 2) -#define ERTS_EV_FLAG_DEFER_OUT_EV ((EventStateFlags) 4) - -#ifdef DEBUG -# define ERTS_ACTIVE_FD_INC 2 -#else -# define ERTS_ACTIVE_FD_INC 128 +#ifndef DEBUG_PRINT_FD +#define DEBUG_PRINT_FD(...) #endif -#define ERTS_CHECK_IO_POLL_RES_LEN 512 +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS +# include "safe_hash.h" +# define DRV_EV_STATE_HTAB_SIZE 1024 +#endif -#if defined(ERTS_KERNEL_POLL_VERSION) -# define ERTS_CIO_EXPORT(FUNC) FUNC ## _kp -#elif defined(ERTS_NO_KERNEL_POLL_VERSION) -# define ERTS_CIO_EXPORT(FUNC) FUNC ## _nkp +typedef enum { + ERTS_EV_TYPE_NONE = 0, + ERTS_EV_TYPE_DRV_SEL = 1, /* driver_select */ + ERTS_EV_TYPE_STOP_USE = 2, /* pending stop_select */ + ERTS_EV_TYPE_NIF = 3, /* enif_select */ + ERTS_EV_TYPE_STOP_NIF = 4 /* pending nif stop */ +} EventStateType; + +typedef enum { + ERTS_EV_FLAG_CLEAR = 0, + ERTS_EV_FLAG_USED = 1, /* ERL_DRV_USE has been turned on */ +#ifdef ERTS_ENABLE_KERNEL_POLL + ERTS_EV_FLAG_FALLBACK = 2, /* Set when kernel poll rejected fd + and it was put in the nkp version */ #else -# define ERTS_CIO_EXPORT(FUNC) FUNC + ERTS_EV_FLAG_FALLBACK = ERTS_EV_FLAG_CLEAR, #endif -#define ERTS_CIO_HAVE_DRV_EVENT \ - (ERTS_POLL_USE_POLL && !ERTS_POLL_USE_KERNEL_POLL) + /* Combinations */ + ERTS_EV_FLAG_USED_FALLBACK = ERTS_EV_FLAG_USED | ERTS_EV_FLAG_FALLBACK +} EventStateFlags; -#define ERTS_CIO_POLL_CTL ERTS_POLL_EXPORT(erts_poll_control) -#define ERTS_CIO_POLL_CTLV ERTS_POLL_EXPORT(erts_poll_controlv) -#define ERTS_CIO_POLL_WAIT ERTS_POLL_EXPORT(erts_poll_wait) -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CIO_POLL_AS_INTR ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt) -#endif -#define ERTS_CIO_POLL_INTR ERTS_POLL_EXPORT(erts_poll_interrupt) -#define ERTS_CIO_POLL_INTR_TMD ERTS_POLL_EXPORT(erts_poll_interrupt_timed) -#define ERTS_CIO_NEW_POLLSET ERTS_POLL_EXPORT(erts_poll_create_pollset) -#define ERTS_CIO_FREE_POLLSET ERTS_POLL_EXPORT(erts_poll_destroy_pollset) -#define ERTS_CIO_POLL_MAX_FDS ERTS_POLL_EXPORT(erts_poll_max_fds) -#define ERTS_CIO_POLL_INIT ERTS_POLL_EXPORT(erts_poll_init) -#define ERTS_CIO_POLL_INFO ERTS_POLL_EXPORT(erts_poll_info) +#define flag2str(flags) \ + ((flags) == ERTS_EV_FLAG_CLEAR ? "CLEAR" : \ + ((flags) == ERTS_EV_FLAG_USED ? "USED" : \ + ((flags) == ERTS_EV_FLAG_FALLBACK ? "FLBK" : \ + ((flags) == ERTS_EV_FLAG_USED_FALLBACK ? "USED|FLBK" : "ERROR")))) -#define GET_FD(fd) fd +/* How many events that can be handled at once by one erts_poll_wait call */ +#define ERTS_CHECK_IO_POLL_RES_LEN 512 -static struct pollset_info +/* Each I/O Poll Thread has one ErtsPollThread each. The ps field + can point to either a private ErtsPollSet or a shared one. + At the moment only kqueue and epoll pollsets can be + shared across threads. +*/ +typedef struct erts_poll_thread { - ErtsPollSet ps; - erts_smp_atomic_t in_poll_wait; /* set while doing poll */ - struct { - int six; /* start index */ - int eix; /* end index */ - erts_smp_atomic32_t no; - int size; - ErtsSysFdType *array; - } active_fd; -#ifdef ERTS_SMP - struct removed_fd* removed_list; /* list of deselected fd's*/ - erts_smp_spinlock_t removed_list_lock; -#endif -}pollset; -#define NUM_OF_POLLSETS 1 + ErtsPollSet *ps; + ErtsPollResFd *pollres; + int pollres_len; +} ErtsPollThread; + +/* pollsetv contains pointers to the ErtsPollSets that are in use. + * Which pollset to use is determined by hashing the fd. + */ +static ErtsPollSet **pollsetv; +#if ERTS_POLL_USE_FALLBACK +static ErtsPollSet *flbk_pollset; +#endif +static ErtsPollThread *psiv; typedef struct { #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS @@ -122,98 +124,151 @@ typedef struct { ErtsSysFdType fd; struct { ErtsDrvSelectDataState *select; /* ERTS_EV_TYPE_DRV_SEL */ -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *event; /* ERTS_EV_TYPE_DRV_EV */ -#endif ErtsNifSelectDataState *nif; /* ERTS_EV_TYPE_NIF */ union { erts_driver_t* drv_ptr; /* ERTS_EV_TYPE_STOP_USE */ ErtsResource* resource; /* ERTS_EV_TYPE_STOP_NIF */ - }stop; + } stop; } driver; - ErtsPollEvents events; - unsigned short remove_cnt; /* number of removed_fd's referring to this fd */ + ErtsPollEvents events; /* The events that have been selected upon */ + ErtsPollEvents active_events; /* The events currently active in the pollset */ EventStateType type; EventStateFlags flags; } ErtsDrvEventState; -#ifdef ERTS_SMP -struct removed_fd { - struct removed_fd *next; +struct drv_ev_state_shared { + + union { + erts_mtx_t lck; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))]; + } locks[ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT]; + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - ErtsSysFdType fd; + int max_fds; + erts_atomic_t len; + ErtsDrvEventState *v; + erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */ #else - ErtsDrvEventState* state; - #ifdef DEBUG - ErtsSysFdType fd; - #endif + SafeHash tab; + int num_prealloc; + ErtsDrvEventState *prealloc_first; + erts_spinlock_t prealloc_lock; #endif - }; -#endif -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static int max_fds = -1; -#endif -#define DRV_EV_STATE_LOCK_CNT 16 -static union { - erts_smp_mtx_t lck; - byte _cache_line_alignment[64]; -}drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT]; +int ERTS_WRITE_UNLIKELY(erts_no_pollsets) = 1; +int ERTS_WRITE_UNLIKELY(erts_no_poll_threads) = 1; +struct drv_ev_state_shared drv_ev_state; -#ifdef ERTS_SMP -static ERTS_INLINE erts_smp_mtx_t* fd_mtx(ErtsSysFdType fd) -{ +static ERTS_INLINE int fd_hash(ErtsSysFdType fd) { int hash = (int)fd; # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS hash ^= (hash >> 9); # endif - return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck; + return hash; +} + +static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) +{ + return &drv_ev_state.locks[fd_hash(fd) % ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT].lck; } -#else -# define fd_mtx(fd) NULL -#endif #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static erts_smp_atomic_t drv_ev_state_len; -static ErtsDrvEventState *drv_ev_state; -static erts_smp_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */ +static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd) +{ + return &drv_ev_state.v[(int) fd]; +} + +#define new_drv_ev_state(State, fd) (State) +#define erase_drv_ev_state(State) -#else -static SafeHash drv_ev_state_tab; -static int num_state_prealloc; -static ErtsDrvEventState *state_prealloc_first; -erts_smp_spinlock_t state_prealloc_lock; +static ERTS_INLINE int grow_drv_ev_state(ErtsSysFdType fd) { + int i; + int old_len; + int new_len; + + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) { -static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd) + if (fd < 0 || fd >= drv_ev_state.max_fds) + return 0; + + erts_mtx_lock(&drv_ev_state.grow_lock); + old_len = erts_atomic_read_nob(&drv_ev_state.len); + if (fd >= old_len) { + new_len = erts_poll_new_table_len(old_len, fd + 1); + if (new_len > drv_ev_state.max_fds) + new_len = drv_ev_state.max_fds; + + for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ + erts_mtx_lock(&drv_ev_state.locks[i].lck); + } + drv_ev_state.v = (drv_ev_state.v + ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, + drv_ev_state.v, + sizeof(ErtsDrvEventState)*new_len) + : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, + sizeof(ErtsDrvEventState)*new_len)); + ERTS_CT_ASSERT(ERTS_EV_TYPE_NONE == 0); + sys_memzero(drv_ev_state.v+old_len, + sizeof(ErtsDrvEventState) * (new_len - old_len)); + for (i = old_len; i < new_len; i++) { + drv_ev_state.v[i].fd = (ErtsSysFdType) i; + } + erts_atomic_set_nob(&drv_ev_state.len, new_len); + for (i=0; i<ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_unlock(&drv_ev_state.locks[i].lck); + } + } + /*else already grown by racing thread */ + + erts_mtx_unlock(&drv_ev_state.grow_lock); + } + return 1; +} + +static int drv_ev_state_len(void) +{ + return erts_atomic_read_nob(&drv_ev_state.len); +} + +#else /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ + +static ERTS_INLINE ErtsDrvEventState *get_drv_ev_state(ErtsSysFdType fd) { ErtsDrvEventState tmpl; tmpl.fd = fd; - return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state_tab, (void *) &tmpl); + return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl); } -static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) +static ERTS_INLINE ErtsDrvEventState* new_drv_ev_state(ErtsDrvEventState *state, + ErtsSysFdType fd) { ErtsDrvEventState tmpl; + + if (state) + return state; + tmpl.fd = fd; tmpl.driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - tmpl.driver.event = NULL; -#endif tmpl.driver.nif = NULL; tmpl.driver.stop.drv_ptr = NULL; tmpl.events = 0; - tmpl.remove_cnt = 0; + tmpl.active_events = 0; tmpl.type = ERTS_EV_TYPE_NONE; tmpl.flags = 0; - return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state_tab, (void *) &tmpl); + + return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl); +} + +static ERTS_INLINE void erase_drv_ev_state(ErtsDrvEventState *state) +{ + safe_hash_erase(&drv_ev_state.tab, (void *) state); } -static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state) +static int drv_ev_state_len(void) { - ASSERT(state->remove_cnt == 0); - safe_hash_erase(&drv_ev_state_tab, (void *) state); + return erts_atomic_read_nob(&drv_ev_state.tab.nitems); } #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ @@ -233,52 +288,47 @@ static void print_nif_select_op(erts_dsprintf_buf_t*, ErtsSysFdType, static void drv_select_large_fd_error(ErlDrvPort, ErtsSysFdType, int, int); static void nif_select_large_fd_error(ErtsSysFdType, int, ErtsResource*,Eterm ref); #endif -#if ERTS_CIO_HAVE_DRV_EVENT -static void drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, - ErlDrvEventData event_data); -static void print_drv_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort, ErtsSysFdType, ErlDrvEventData); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void event_large_fd_error(ErlDrvPort, ErtsSysFdType, ErlDrvEventData); -#endif -#endif static void steal_pending_stop_use(erts_dsprintf_buf_t*, ErlDrvPort, ErtsDrvEventState*, int mode, int on); static void steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource*, ErtsDrvEventState *state, int mode, int on); - -#ifdef ERTS_SMP -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(removed_fd, struct removed_fd, 64, ERTS_ALC_T_FD_LIST) +static ERTS_INLINE void +check_fd_cleanup(ErtsDrvEventState *state, + ErtsDrvSelectDataState **free_select, + ErtsNifSelectDataState **free_nif); +static ERTS_INLINE void iready(Eterm id, ErtsDrvEventState *state); +static ERTS_INLINE void oready(Eterm id, ErtsDrvEventState *state); +#ifdef DEBUG_PRINT_MODE +static char *drvmode2str(int mode); +static char *nifmode2str(enum ErlNifSelectFlags mode); #endif static ERTS_INLINE void -init_iotask(ErtsIoTask *io_task) +init_iotask(ErtsIoTask *io_task, ErtsSysFdType fd) { erts_port_task_handle_init(&io_task->task); - erts_smp_atomic_init_nob(&io_task->executed_time, ~((erts_aint_t) 0)); + io_task->fd = fd; } static ERTS_INLINE int -is_iotask_active(ErtsIoTask *io_task, erts_aint_t current_cio_time) -{ +is_iotask_active(ErtsIoTask *io_task) +{ if (erts_port_task_is_scheduled(&io_task->task)) return 1; - if (erts_smp_atomic_read_nob(&io_task->executed_time) == current_cio_time) - return 1; return 0; } static ERTS_INLINE ErtsDrvSelectDataState * -alloc_drv_select_data(void) +alloc_drv_select_data(ErtsSysFdType fd) { ErtsDrvSelectDataState *dsp = erts_alloc(ERTS_ALC_T_DRV_SEL_D_STATE, sizeof(ErtsDrvSelectDataState)); dsp->inport = NIL; dsp->outport = NIL; - init_iotask(&dsp->iniotask); - init_iotask(&dsp->outiotask); + init_iotask(&dsp->iniotask, fd); + init_iotask(&dsp->outiotask, fd); return dsp; } @@ -289,8 +339,6 @@ alloc_nif_select_data(void) sizeof(ErtsNifSelectDataState)); dsp->in.pid = NIL; dsp->out.pid = NIL; - dsp->in.ddeselect_cnt = 0; - dsp->out.ddeselect_cnt = 0; return dsp; } @@ -308,209 +356,136 @@ free_nif_select_data(ErtsNifSelectDataState *dsp) erts_free(ERTS_ALC_T_NIF_SEL_D_STATE, dsp); } -#if ERTS_CIO_HAVE_DRV_EVENT - -static ERTS_INLINE ErtsDrvEventDataState * -alloc_drv_event_data(void) +static ERTS_INLINE int +get_pollset_id(ErtsSysFdType fd) { - ErtsDrvEventDataState *dep = erts_alloc(ERTS_ALC_T_DRV_EV_D_STATE, - sizeof(ErtsDrvEventDataState)); - dep->port = NIL; - dep->data = NULL; - dep->removed_events = 0; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - dep->deferred_events = 0; -#endif - init_iotask(&dep->iotask); - return dep; + return fd_hash(fd) % erts_no_pollsets; } -static ERTS_INLINE void -free_drv_event_data(ErtsDrvEventDataState *dep) +static ERTS_INLINE ErtsPollSet * +get_pollset(ErtsSysFdType fd) { - ASSERT(!erts_port_task_is_scheduled(&dep->iotask.task)); - erts_free(ERTS_ALC_T_DRV_EV_D_STATE, dep); + return pollsetv[get_pollset_id(fd)]; } -#endif /* ERTS_CIO_HAVE_DRV_EVENT */ - -static ERTS_INLINE void -remember_removed(ErtsDrvEventState *state, struct pollset_info* psi) +#if ERTS_POLL_USE_FALLBACK +static ERTS_INLINE ErtsPollSet * +get_fallback(void) { -#ifdef ERTS_SMP - struct removed_fd *fdlp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) { - state->remove_cnt++; - ASSERT(state->remove_cnt > 0); - fdlp = removed_fd_alloc(); - #if defined(ERTS_SYS_CONTINOUS_FD_NUMBERS) || defined(DEBUG) - fdlp->fd = state->fd; - #endif - #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - fdlp->state = state; - #endif - erts_smp_spin_lock(&psi->removed_list_lock); - fdlp->next = psi->removed_list; - psi->removed_list = fdlp; - erts_smp_spin_unlock(&psi->removed_list_lock); - } -#endif + return flbk_pollset; } +#endif - -static ERTS_INLINE int -is_removed(ErtsDrvEventState *state) +/* + * Place a fd within a pollset. This will automatically use + * the fallback ps if needed. + */ +static ERTS_INLINE ErtsPollEvents +erts_io_control_wakeup(ErtsDrvEventState *state, ErtsPollOp op, + ErtsPollEvents pe, int *wake_poller) { -#ifdef ERTS_SMP - /* Note that there is a possible race here, where an fd is removed - (increasing remove_cnt) and then added again just before erts_poll_wait - is called by erts_check_io. Any polled event on the re-added fd will then - be falsely ignored. But that does not matter, as the event will trigger - again next time erl_check_io is called. */ - return state->remove_cnt > 0; -#else - return 0; + ErtsSysFdType fd = state->fd; + ErtsPollEvents res = 0; + EventStateFlags flags = state->flags; + + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); + + if (!(flags & ERTS_EV_FLAG_FALLBACK)) { + res = erts_poll_control(get_pollset(fd), fd, op, pe, wake_poller); + +#if ERTS_POLL_USE_FALLBACK + if (op == ERTS_POLL_OP_ADD && res == ERTS_POLL_EV_NVAL) { + /* When an add fails with NVAL, the poll/kevent operation could not + put that fd in the pollset, so we instead put it into a fallback pollset */ + state->flags |= ERTS_EV_FLAG_FALLBACK; + res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller); + } + } else { + ASSERT(op != ERTS_POLL_OP_ADD); + res = erts_poll_control_flbk(get_fallback(), fd, op, pe, wake_poller); #endif + } + + return res; } -static void -forget_removed(struct pollset_info* psi) +static ERTS_INLINE ErtsPollEvents +erts_io_control(ErtsDrvEventState *state, ErtsPollOp op, ErtsPollEvents pe) { -#ifdef ERTS_SMP - struct removed_fd* fdlp; - struct removed_fd* tofree; + int wake_poller = 0; + return erts_io_control_wakeup(state, op, pe, &wake_poller); +} + +/* ToDo: Was inline in erl_check_io.h but now need struct erts_poll_thread */ +void +erts_io_notify_port_task_executed(ErtsPortTaskType type, + ErtsPortTaskHandle *pthp, + void (*reset_handle)(ErtsPortTaskHandle *)) +{ + ErtsIoTask *itp = ErtsContainerStruct(pthp, ErtsIoTask, task); + ErtsSysFdType fd = itp->fd; + erts_mtx_t *mtx = fd_mtx(fd); + int active_events; + ErtsDrvEventState *state; + ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; - /* Fast track: if (atomic_ptr(removed_list)==NULL) return; */ + erts_mtx_lock(mtx); + state = get_drv_ev_state(fd); - erts_smp_spin_lock(&psi->removed_list_lock); - fdlp = psi->removed_list; - psi->removed_list = NULL; - erts_smp_spin_unlock(&psi->removed_list_lock); + active_events = state->active_events; - while (fdlp) { - ErtsResource* resource = NULL; - erts_driver_t* drv_ptr = NULL; - erts_smp_mtx_t* mtx; - ErtsSysFdType fd; - ErtsDrvEventState *state; + switch (type) { + case ERTS_PORT_TASK_INPUT: -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - fd = fdlp->fd; - mtx = fd_mtx(fd); - erts_smp_mtx_lock(mtx); - state = &drv_ev_state[(int) fd]; -#else - state = fdlp->state; - fd = state->fd; - ASSERT(fd == fdlp->fd); - mtx = fd_mtx(fd); - erts_smp_mtx_lock(mtx); -#endif - ASSERT(state->remove_cnt > 0); - if (--state->remove_cnt == 0) { - switch (state->type) { - case ERTS_EV_TYPE_STOP_NIF: - /* Now we can call stop */ - resource = state->driver.stop.resource; - state->driver.stop.resource = NULL; - ASSERT(resource); - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - goto case_ERTS_EV_TYPE_NONE; - - case ERTS_EV_TYPE_STOP_USE: - /* Now we can call stop_select */ - drv_ptr = state->driver.stop.drv_ptr; - ASSERT(drv_ptr); - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.stop.drv_ptr = NULL; - /* Fall through */ - case ERTS_EV_TYPE_NONE: - case_ERTS_EV_TYPE_NONE: -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - hash_erase_drv_ev_state(state); -#endif - break; - case ERTS_EV_TYPE_DRV_SEL: - case ERTS_EV_TYPE_DRV_EV: - break; - default: - ASSERT(0); - } - } - erts_smp_mtx_unlock(mtx); - if (drv_ptr) { - int was_unmasked = erts_block_fpe(); - DTRACE1(driver_stop_select, drv_ptr->name); - LTTNG1(driver_stop_select, drv_ptr->name); - (*drv_ptr->stop_select) ((ErlDrvEvent) fd, NULL); - erts_unblock_fpe(was_unmasked); - if (drv_ptr->handle) { - erts_ddll_dereference_driver(drv_ptr->handle); - } - } - if (resource) { - erts_resource_stop(resource, (ErlNifEvent)fd, 0); - enif_release_resource(resource->data); - } + DEBUG_PRINT_FD("executed ready_input", state); - tofree = fdlp; - fdlp = fdlp->next; - removed_fd_free(tofree); + ASSERT(!(state->active_events & ERTS_POLL_EV_IN)); + if (state->events & ERTS_POLL_EV_IN) + active_events |= ERTS_POLL_EV_IN; + break; + case ERTS_PORT_TASK_OUTPUT: + + DEBUG_PRINT_FD("executed ready_output", state); + + ASSERT(!(state->active_events & ERTS_POLL_EV_OUT)); + if (state->events & ERTS_POLL_EV_OUT) + active_events |= ERTS_POLL_EV_OUT; + break; + default: + erts_exit(ERTS_ABORT_EXIT, "Invalid IO port task type"); + break; } -#endif /* ERTS_SMP */ -} -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void -grow_drv_ev_state(int min_ix) -{ - int i; - int old_len; - int new_len; + reset_handle(pthp); - erts_smp_mtx_lock(&drv_ev_state_grow_lock); - old_len = erts_smp_atomic_read_nob(&drv_ev_state_len); - if (min_ix >= old_len) { - new_len = erts_poll_new_table_len(old_len, min_ix + 1); - if (new_len > max_fds) - new_len = max_fds; + if (active_events) { + /* This is not needed if active_events has not changed */ + if (state->active_events != active_events) { + ErtsPollEvents new_events; + state->active_events = active_events; + new_events = erts_io_control(state, ERTS_POLL_OP_MOD, active_events); - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ - erts_smp_mtx_lock(&drv_ev_state_locks[i].lck); - } - drv_ev_state = (drv_ev_state - ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, - drv_ev_state, - sizeof(ErtsDrvEventState)*new_len) - : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, - sizeof(ErtsDrvEventState)*new_len)); - for (i = old_len; i < new_len; i++) { - drv_ev_state[i].fd = (ErtsSysFdType) i; - drv_ev_state[i].driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - drv_ev_state[i].driver.event = NULL; -#endif - drv_ev_state[i].driver.stop.drv_ptr = NULL; - drv_ev_state[i].driver.nif = NULL; - drv_ev_state[i].events = 0; - drv_ev_state[i].remove_cnt = 0; - drv_ev_state[i].type = ERTS_EV_TYPE_NONE; - drv_ev_state[i].flags = 0; - } - erts_smp_atomic_set_nob(&drv_ev_state_len, new_len); - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck); - } + /* We were unable to re-insert the fd into the pollset, signal the callback. */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (active_events & ERTS_POLL_EV_IN) + iready(state->driver.select->inport, state); + if (active_events & ERTS_POLL_EV_OUT) + oready(state->driver.select->outport, state); + state->active_events = 0; + } + } + } else { + check_fd_cleanup(state, &free_select, &free_nif); } - /*else already grown by racing thread */ - erts_smp_mtx_unlock(&drv_ev_state_grow_lock); -} -#endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */ + erts_mtx_unlock(mtx); + if (free_select) + free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); +} static ERTS_INLINE void abort_task(Eterm id, ErtsPortTaskHandle *pthp, EventStateType type) @@ -527,13 +502,6 @@ abort_tasks(ErtsDrvEventState *state, int mode) switch (mode) { case 0: check_type: switch (state->type) { -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - abort_task(state->driver.event->port, - &state->driver.event->iotask.task, - ERTS_EV_TYPE_DRV_EV); - return; -#endif case ERTS_EV_TYPE_NIF: case ERTS_EV_TYPE_NONE: return; @@ -563,16 +531,14 @@ abort_tasks(ErtsDrvEventState *state, int mode) static void deselect(ErtsDrvEventState *state, int mode) { - int do_wake = 0; ErtsPollEvents rm_events; - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - ASSERT(state->events); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); abort_tasks(state, mode); - if (!mode) + if (!mode) { rm_events = state->events; - else { + } else { rm_events = 0; ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); if (mode & ERL_DRV_READ) { @@ -585,67 +551,63 @@ deselect(ErtsDrvEventState *state, int mode) } } - state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, rm_events, 0, &do_wake); + state->events &= ~rm_events; + state->active_events &= ~rm_events; if (!(state->events)) { + erts_io_control(state, ERTS_POLL_OP_DEL, 0); switch (state->type) { case ERTS_EV_TYPE_NIF: state->driver.nif->in.pid = NIL; state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - enif_release_resource(state->driver.stop.resource); + enif_release_resource(state->driver.stop.resource->data); state->driver.stop.resource = NULL; break; case ERTS_EV_TYPE_DRV_SEL: state->driver.select->inport = NIL; state->driver.select->outport = NIL; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - state->driver.event->port = NIL; - state->driver.event->data = NULL; - state->driver.event->removed_events = (ErtsPollEvents) 0; - break; -#endif case ERTS_EV_TYPE_NONE: break; default: ASSERT(0); break; } - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - remember_removed(state, &pollset); + state->flags = 0; + } else { + ErtsPollEvents new_events = + erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events); + + /* We were unable to re-insert the fd into the pollset, signal the callback. */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (state->active_events & ERTS_POLL_EV_IN) + iready(state->driver.select->inport, state); + if (state->active_events & ERTS_POLL_EV_OUT) + oready(state->driver.select->outport, state); + state->active_events = 0; + } } } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -# define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE && (state)->remove_cnt == 0) +# define IS_FD_UNKNOWN(state) ((state)->type == ERTS_EV_TYPE_NONE) #else # define IS_FD_UNKNOWN(state) ((state) == NULL) #endif static ERTS_INLINE void check_fd_cleanup(ErtsDrvEventState *state, -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState **free_event, -#endif ErtsDrvSelectDataState **free_select, ErtsNifSelectDataState **free_nif) { - erts_aint_t current_cio_time; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd))); - - current_cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); + ERTS_LC_ASSERT(erts_lc_mtx_is_locked(fd_mtx(state->fd))); *free_select = NULL; if (state->driver.select && (state->type != ERTS_EV_TYPE_DRV_SEL) - && !is_iotask_active(&state->driver.select->iniotask, current_cio_time) - && !is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { - + && !is_iotask_active(&state->driver.select->iniotask) + && !is_iotask_active(&state->driver.select->outiotask)) { + *free_select = state->driver.select; state->driver.select = NULL; } @@ -656,382 +618,85 @@ check_fd_cleanup(ErtsDrvEventState *state, state->driver.nif = NULL; } -#if ERTS_CIO_HAVE_DRV_EVENT - *free_event = NULL; - if (state->driver.event - && (state->type != ERTS_EV_TYPE_DRV_EV) - && !is_iotask_active(&state->driver.event->iotask, current_cio_time)) { - - *free_event = state->driver.event; - state->driver.event = NULL; - } -#endif - -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS if (((state->type != ERTS_EV_TYPE_NONE) - | state->remove_cnt -#if ERTS_CIO_HAVE_DRV_EVENT - | (state->driver.event != NULL) -#endif + | (state->driver.nif != NULL) | (state->driver.select != NULL)) == 0) { - hash_erase_drv_ev_state(state); - + erase_drv_ev_state(state); } -#endif } -static ERTS_INLINE int -check_cleanup_active_fd(ErtsSysFdType fd, -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollControlEntry *pce, - int *pce_ix, -#endif - erts_aint_t current_cio_time, - int may_sleep) -{ - ErtsDrvEventState *state; - int active = 0; - erts_smp_mtx_t *mtx = fd_mtx(fd); - void *free_select = NULL; - void *free_nif = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - void *free_event = NULL; -#endif -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents evon = 0, evoff = 0; -#endif - - erts_smp_mtx_lock(mtx); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; +#ifdef __WIN32__ +# define MUST_DEFER(MAY_SLEEP) 1 #else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ - if (state) +# define MUST_DEFER(MAY_SLEEP) (MAY_SLEEP) #endif - { - if (state->driver.select) { -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - if (is_iotask_active(&state->driver.select->iniotask, current_cio_time)) { - active = 1; - if ((state->events & ERTS_POLL_EV_IN) - && !(state->flags & ERTS_EV_FLAG_DEFER_IN_EV)) { - evoff |= ERTS_POLL_EV_IN; - state->flags |= ERTS_EV_FLAG_DEFER_IN_EV; - } - } - else if (state->flags & ERTS_EV_FLAG_DEFER_IN_EV) { - if (state->events & ERTS_POLL_EV_IN) - evon |= ERTS_POLL_EV_IN; - state->flags &= ~ERTS_EV_FLAG_DEFER_IN_EV; - } - if (is_iotask_active(&state->driver.select->outiotask, current_cio_time)) { - active = 1; - if ((state->events & ERTS_POLL_EV_OUT) - && !(state->flags & ERTS_EV_FLAG_DEFER_OUT_EV)) { - evoff |= ERTS_POLL_EV_OUT; - state->flags |= ERTS_EV_FLAG_DEFER_OUT_EV; - } - } - else if (state->flags & ERTS_EV_FLAG_DEFER_OUT_EV) { - if (state->events & ERTS_POLL_EV_OUT) - evon |= ERTS_POLL_EV_OUT; - state->flags &= ~ERTS_EV_FLAG_DEFER_OUT_EV; - } - if (active) - (void) 0; - else -#else - if (is_iotask_active(&state->driver.select->iniotask, current_cio_time) - || is_iotask_active(&state->driver.select->outiotask, current_cio_time)) - active = 1; - else -#endif - if (state->type != ERTS_EV_TYPE_DRV_SEL) { - free_select = state->driver.select; - state->driver.select = NULL; - } - } - - if (state->driver.nif) { - ErtsPollEvents rm_events = 0; - if (state->driver.nif->in.ddeselect_cnt) { - ASSERT(state->type == ERTS_EV_TYPE_NIF); - ASSERT(state->events & ERTS_POLL_EV_IN); - ASSERT(is_nil(state->driver.nif->in.pid)); - if (may_sleep || state->driver.nif->in.ddeselect_cnt == 1) { - rm_events = ERTS_POLL_EV_IN; - state->driver.nif->in.ddeselect_cnt = 0; - } - } - if (state->driver.nif->out.ddeselect_cnt) { - ASSERT(state->type == ERTS_EV_TYPE_NIF); - ASSERT(state->events & ERTS_POLL_EV_OUT); - ASSERT(is_nil(state->driver.nif->out.pid)); - if (may_sleep || state->driver.nif->out.ddeselect_cnt == 1) { - rm_events |= ERTS_POLL_EV_OUT; - state->driver.nif->out.ddeselect_cnt = 0; - } - } - if (rm_events) { - int do_wake = 0; - state->events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, - rm_events, 0, &do_wake); - } - if (state->events) - active = 1; - else if (state->type != ERTS_EV_TYPE_NIF) { - free_nif = state->driver.nif; - state->driver.nif = NULL; - } - } - -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) { - if (is_iotask_active(&state->driver.event->iotask, current_cio_time)) { -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents evs = state->events & ~state->driver.event->deferred_events; - if (evs) { - evoff |= evs; - state->driver.event->deferred_events |= evs; - } -#endif - active = 1; - } - else if (state->type != ERTS_EV_TYPE_DRV_EV) { - free_event = state->driver.event; - state->driver.event = NULL; - } -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - else { - ErtsPollEvents evs = state->events & state->driver.event->deferred_events; - if (evs) { - evon |= evs; - state->driver.event->deferred_events = 0; - } - } -#endif - - } -#endif - -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (((state->type != ERTS_EV_TYPE_NONE) | state->remove_cnt | active) == 0) - hash_erase_drv_ev_state(state); -#endif - - } - - erts_smp_mtx_unlock(mtx); - - if (free_select) - free_drv_select_data(free_select); - if (free_nif) - free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif - -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - if (evoff) { - ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; - pcep->fd = fd; - pcep->events = evoff; - pcep->on = 0; - } - if (evon) { - ErtsPollControlEntry *pcep = &pce[(*pce_ix)++]; - pcep->fd = fd; - pcep->events = evon; - pcep->on = 1; - } -#endif - - return active; -} - -static void -check_cleanup_active_fds(erts_aint_t current_cio_time, int may_sleep) -{ - int six = pollset.active_fd.six; - int eix = pollset.active_fd.eix; - erts_aint32_t no = erts_smp_atomic32_read_dirty(&pollset.active_fd.no); - int size = pollset.active_fd.size; - int ix = six; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - /* every fd might add two entries */ - Uint pce_sz = 2*sizeof(ErtsPollControlEntry)*no; - ErtsPollControlEntry *pctrl_entries = (pce_sz - ? erts_alloc(ERTS_ALC_T_TMP, pce_sz) - : NULL); - int pctrl_ix = 0; -#endif - - while (ix != eix) { - ErtsSysFdType fd = pollset.active_fd.array[ix]; - int nix = ix + 1; - if (nix >= size) - nix = 0; - ASSERT(fd != ERTS_SYS_FD_INVALID); - if (!check_cleanup_active_fd(fd, -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - pctrl_entries, - &pctrl_ix, -#endif - current_cio_time, - may_sleep)) { - no--; - if (ix == six) { -#ifdef DEBUG - pollset.active_fd.array[ix] = ERTS_SYS_FD_INVALID; -#endif - six = nix; - } - else { - pollset.active_fd.array[ix] = pollset.active_fd.array[six]; -#ifdef DEBUG - pollset.active_fd.array[six] = ERTS_SYS_FD_INVALID; -#endif - six++; - if (six >= size) - six = 0; - } - } - ix = nix; - } - -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ASSERT(pctrl_ix <= pce_sz/sizeof(ErtsPollControlEntry)); - if (pctrl_ix) - ERTS_CIO_POLL_CTLV(pollset.ps, pctrl_entries, pctrl_ix); - if (pctrl_entries) - erts_free(ERTS_ALC_T_TMP, pctrl_entries); -#endif - - pollset.active_fd.six = six; - pollset.active_fd.eix = eix; - erts_smp_atomic32_set_relb(&pollset.active_fd.no, no); -} - -static void grow_active_fds(void) -{ - ASSERT(pollset.active_fd.six == pollset.active_fd.eix); - pollset.active_fd.six = 0; - pollset.active_fd.eix = pollset.active_fd.size; - pollset.active_fd.size += ERTS_ACTIVE_FD_INC; - pollset.active_fd.array = erts_realloc(ERTS_ALC_T_ACTIVE_FD_ARR, - pollset.active_fd.array, - pollset.active_fd.size*sizeof(ErtsSysFdType)); -#ifdef DEBUG - { - int i; - for (i = pollset.active_fd.eix + 1; i < pollset.active_fd.size; i++) - pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; - } -#endif -} - -static ERTS_INLINE void -add_active_fd(ErtsSysFdType fd) -{ - int eix = pollset.active_fd.eix; - int size = pollset.active_fd.size; - - pollset.active_fd.array[eix] = fd; - - erts_smp_atomic32_set_relb(&pollset.active_fd.no, - (erts_smp_atomic32_read_dirty(&pollset.active_fd.no) - + 1)); - - eix++; - if (eix >= size) - eix = 0; - pollset.active_fd.eix = eix; - - if (pollset.active_fd.six == eix) { - grow_active_fds(); - } -} int -ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, - ErlDrvEvent e, - int mode, - int on) +driver_select(ErlDrvPort ix, ErlDrvEvent e, int mode, int on) { void (*stop_select_fn)(ErlDrvEvent, void*) = NULL; Port *prt = erts_drvport2port(ix); Eterm id = erts_drvport2id(ix); ErtsSysFdType fd = (ErtsSysFdType) e; ErtsPollEvents ctl_events = (ErtsPollEvents) 0; - ErtsPollEvents new_events, old_events; + ErtsPollEvents old_events; + ErtsPollEvents new_events; + ErtsPollOp ctl_op = ERTS_POLL_OP_MOD; ErtsDrvEventState *state; - int wake_poller; + int wake_poller = 0; int ret; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event = NULL; -#endif ErtsDrvSelectDataState *free_select = NULL; ErtsNifSelectDataState *free_nif = NULL; #ifdef USE_VM_PROBES DTRACE_CHARBUF(name, 64); #endif + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO); - if (prt == ERTS_INVALID_ERL_DRV_PORT) + if (prt == ERTS_INVALID_ERL_DRV_PORT) { + ERTS_MSACC_POP_STATE(); return -1; + } - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) { - return -1; - } - if (fd >= max_fds) { - drv_select_large_fd_error(ix, fd, mode, on); - return -1; - } - grow_drv_ev_state(fd); + if (!grow_drv_ev_state(fd)) { + if (fd > 0) drv_select_large_fd_error(ix, fd, mode, on); + ERTS_MSACC_POP_STATE(); + return -1; } #endif - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ -#endif + state = get_drv_ev_state(fd); /* may be NULL! */ - if (!on && (mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - if (IS_FD_UNKNOWN(state)) { - /* fast track to stop_select callback */ - stop_select_fn = prt->drv_ptr->stop_select; -#ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, - sizeof(DTRACE_CHARBUF_NAME(name))-1); - name[sizeof(name)-1] = '\0'; -#endif - ret = 0; - goto done_unknown; - } - mode |= (ERL_DRV_READ | ERL_DRV_WRITE); - wake_poller = 1; /* to eject fd from pollset (if needed) */ - } - else wake_poller = 0; + DEBUG_PRINT_FD("driver_select(%T, %p, %s, %d)", + state, id, fd, drvmode2str(mode), on); -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state == NULL) { - state = hash_new_drv_ev_state(fd); + if (!on) { + if (IS_FD_UNKNOWN(state)) { + if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + /* fast track to stop_select callback */ + stop_select_fn = prt->drv_ptr->stop_select; + #ifdef USE_VM_PROBES + strncpy(name, prt->drv_ptr->name, + sizeof(DTRACE_CHARBUF_NAME(name))-1); + name[sizeof(name)-1] = '\0'; + #endif + } + ret = 0; + goto done_unknown; + } + else if ((mode&ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + mode |= (ERL_DRV_READ | ERL_DRV_WRITE); + } } -#endif + + state = new_drv_ev_state(state, fd); switch (state->type) { -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: -#endif case ERTS_EV_TYPE_NIF: drv_select_steal(ix, state, mode, on); break; @@ -1053,7 +718,9 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ASSERT(state->type == ERTS_EV_TYPE_NONE); break; - }} + } + default: break; + } if (mode & ERL_DRV_READ) { if (state->type == ERTS_EV_TYPE_DRV_SEL) { @@ -1061,7 +728,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, if (owner != id && is_not_nil(owner)) drv_select_steal(ix, state, mode, on); } - ctl_events |= ERTS_POLL_EV_IN; + ctl_events = ERTS_POLL_EV_IN; } if (mode & ERL_DRV_WRITE) { if (state->type == ERTS_EV_TYPE_DRV_SEL) { @@ -1070,100 +737,105 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, drv_select_steal(ix, state, mode, on); } ctl_events |= ERTS_POLL_EV_OUT; - } + } + ASSERT((state->type == ERTS_EV_TYPE_DRV_SEL) || (state->type == ERTS_EV_TYPE_NONE && !state->events)); - if (!on && !(state->flags & ERTS_EV_FLAG_USED) - && state->events && !(state->events & ~ctl_events)) { - /* Old driver removing all events. At least wake poller. - It will not make close() 100% safe but it will prevent - actions delayed by poll timeout. */ - wake_poller = 1; - } + old_events = state->events; - new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller); + if (on) { + ctl_events &= ~old_events; + state->events |= ctl_events; + if (ctl_events & ERTS_POLL_EV_IN && (!state->driver.select || !is_iotask_active(&state->driver.select->iniotask))) + state->active_events |= ERTS_POLL_EV_IN; + if (ctl_events & ERTS_POLL_EV_OUT && (!state->driver.select || !is_iotask_active(&state->driver.select->outiotask))) + state->active_events |= ERTS_POLL_EV_OUT; + if (old_events == 0 && !(state->flags & ERTS_EV_FLAG_USED)) { + ctl_op = ERTS_POLL_OP_ADD; + } + } + else { + ctl_events &= old_events; + state->events &= ~ctl_events; + state->active_events &= ~ctl_events; - if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - if (state->type == ERTS_EV_TYPE_DRV_SEL && !state->events) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.select->inport = NIL; - state->driver.select->outport = NIL; - } - ret = -1; - goto done; + if (!state->events) { + if (!(state->flags & ERTS_EV_FLAG_USED) || mode & ERL_DRV_USE) + ctl_op = ERTS_POLL_OP_DEL; + } } - old_events = state->events; + if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) { - ASSERT(on - ? (new_events == (state->events | ctl_events)) - : (new_events == (state->events & ~ctl_events))); + new_events = erts_io_control_wakeup(state, ctl_op, + state->active_events, + &wake_poller); - ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL - || state->type == ERTS_EV_TYPE_NONE); + ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL || state->type == ERTS_EV_TYPE_NONE); + } - state->events = new_events; - if (ctl_events) { - if (on) { + if (on) { + if (ctl_events) { if (!state->driver.select) - state->driver.select = alloc_drv_select_data(); + state->driver.select = alloc_drv_select_data(state->fd); if (state->type == ERTS_EV_TYPE_NONE) state->type = ERTS_EV_TYPE_DRV_SEL; ASSERT(state->type == ERTS_EV_TYPE_DRV_SEL); - if (ctl_events & ERTS_POLL_EV_IN) + if (ctl_events & ERTS_POLL_EV_IN) { state->driver.select->inport = id; - if (ctl_events & ERTS_POLL_EV_OUT) + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) + iready(id, state); + } + if (ctl_events & ERTS_POLL_EV_OUT) { state->driver.select->outport = id; - if (mode & ERL_DRV_USE) { + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) + oready(id, state); + } + if (mode & ERL_DRV_USE) state->flags |= ERTS_EV_FLAG_USED; - } - } - else { /* off */ - if (state->type == ERTS_EV_TYPE_DRV_SEL) { - if (ctl_events & ERTS_POLL_EV_IN) { - abort_tasks(state, ERL_DRV_READ); - state->driver.select->inport = NIL; - } - if (ctl_events & ERTS_POLL_EV_OUT) { - abort_tasks(state, ERL_DRV_WRITE); - state->driver.select->outport = NIL; - } - if (new_events == 0) { - if (old_events != 0) { - remember_removed(state, &pollset); - } - if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - } - /*else keep it, as fd will probably be selected upon again */ - } - } - if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { - erts_driver_t* drv_ptr = prt->drv_ptr; - ASSERT(new_events==0); - if (state->remove_cnt == 0 || !wake_poller) { - /* Safe to close fd now as it is not in pollset - or there was no need to eject fd (kernel poll) */ - stop_select_fn = drv_ptr->stop_select; + } + } + else { /* off */ + if (state->type == ERTS_EV_TYPE_DRV_SEL) { + if (ctl_events & ERTS_POLL_EV_IN) { + abort_tasks(state, ERL_DRV_READ); + state->driver.select->inport = NIL; + } + if (ctl_events & ERTS_POLL_EV_OUT) { + abort_tasks(state, ERL_DRV_WRITE); + state->driver.select->outport = NIL; + } + if (state->events == 0) { + if ((mode & ERL_DRV_USE) || !(state->flags & ERTS_EV_FLAG_USED)) { + state->type = ERTS_EV_TYPE_NONE; + state->flags = 0; + } + /*else keep it, as fd will probably be selected upon again */ + } + } + if ((mode & ERL_DRV_USE_NO_CALLBACK) == ERL_DRV_USE) { + erts_driver_t* drv_ptr = prt->drv_ptr; + ASSERT(state->events==0); + if (!wake_poller) { + /* Safe to close fd now as it is not in pollset + or there was no need to eject fd (kernel poll) */ + stop_select_fn = drv_ptr->stop_select; #ifdef USE_VM_PROBES - strncpy(name, prt->drv_ptr->name, sizeof(name)-1); - name[sizeof(name)-1] = '\0'; + strncpy(name, prt->drv_ptr->name, sizeof(name)-1); + name[sizeof(name)-1] = '\0'; #endif - } - else { - /* Not safe to close fd, postpone stop_select callback. */ - state->type = ERTS_EV_TYPE_STOP_USE; - state->driver.stop.drv_ptr = drv_ptr; - if (drv_ptr->handle) { - erts_ddll_reference_referenced_driver(drv_ptr->handle); - } - } - } - } + } + else { + /* Not safe to close fd, postpone stop_select callback. */ + state->type = ERTS_EV_TYPE_STOP_USE; + state->driver.stop.drv_ptr = drv_ptr; + if (drv_ptr->handle) { + erts_ddll_reference_referenced_driver(drv_ptr->handle); + } + } + } } ret = 0; @@ -1171,14 +843,11 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, done: check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif &free_select, &free_nif); done_unknown: - erts_smp_mtx_unlock(fd_mtx(fd)); + erts_mtx_unlock(fd_mtx(fd)); if (stop_select_fn) { int was_unmasked = erts_block_fpe(); DTRACE1(driver_stop_select, name); @@ -1191,61 +860,47 @@ done_unknown: if (free_nif) free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif + ERTS_MSACC_POP_STATE(); + return ret; } int -ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, - ErlNifEvent e, - enum ErlNifSelectFlags mode, - void* obj, - const ErlNifPid* pid, - Eterm ref) +enif_select(ErlNifEnv* env, + ErlNifEvent e, + enum ErlNifSelectFlags mode, + void* obj, + const ErlNifPid* pid, + Eterm ref) { int on; ErtsResource* resource = DATA_TO_RESOURCE(obj); ErtsSysFdType fd = (ErtsSysFdType) e; ErtsPollEvents ctl_events = (ErtsPollEvents) 0; - ErtsPollEvents new_events, old_events; + ErtsPollEvents old_events; + ErtsPollOp ctl_op = ERTS_POLL_OP_MOD; ErtsDrvEventState *state; - int wake_poller; - int ret; + int ret, wake_poller = 0; enum { NO_STOP=0, CALL_STOP, CALL_STOP_AND_RELEASE } call_stop = NO_STOP; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event = NULL; -#endif ErtsDrvSelectDataState *free_select = NULL; ErtsNifSelectDataState *free_nif = NULL; -#ifdef USE_VM_PROBES - DTRACE_CHARBUF(name, 64); -#endif ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) { - return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; - } - if (fd >= max_fds) { - nif_select_large_fd_error(fd, mode, resource, ref); - return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; - } - grow_drv_ev_state(fd); + if (!grow_drv_ev_state(fd)) { + if (fd > 0) nif_select_large_fd_error(fd, mode, resource, ref); + return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } #endif - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - state = hash_get_drv_ev_state(fd); /* may be NULL! */ -#endif + state = get_drv_ev_state(fd); /* may be NULL! */ + + DEBUG_PRINT_FD("enif_select(%T, %d, %s, %p, %T, %T)", + state, env->proc->common.id, fd, nifmode2str(mode), resource, + pid ? pid->pid : THE_NON_VALUE, ref); if (mode & ERL_NIF_SELECT_STOP) { ASSERT(resource->type->stop); @@ -1257,13 +912,12 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } on = 0; mode = ERL_DRV_READ | ERL_DRV_WRITE | ERL_DRV_USE; - wake_poller = 1; /* to eject fd from pollset (if needed) */ ctl_events = ERTS_POLL_EV_IN | ERTS_POLL_EV_OUT; + ctl_op = ERTS_POLL_OP_DEL; } else { on = 1; ASSERT(mode); - wake_poller = 0; if (mode & ERL_DRV_READ) { ctl_events |= ERTS_POLL_EV_IN; } @@ -1272,11 +926,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } } -#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state == NULL) { - state = hash_new_drv_ev_state(fd); - } -#endif + state = new_drv_ev_state(state,fd); switch (state->type) { case ERTS_EV_TYPE_NIF: @@ -1287,9 +937,6 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, if (state->driver.stop.resource != resource) nif_select_steal(state, ERL_DRV_READ | ERL_DRV_WRITE, resource, ref); break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: -#endif case ERTS_EV_TYPE_DRV_SEL: nif_select_steal(state, mode, resource, ref); break; @@ -1310,37 +957,52 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } ASSERT(state->type == ERTS_EV_TYPE_NONE); break; - }} + } + default: break; + } ASSERT((state->type == ERTS_EV_TYPE_NIF) || (state->type == ERTS_EV_TYPE_NONE && !state->events)); - new_events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, ctl_events, on, &wake_poller); + old_events = state->events; - if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - if (state->type == ERTS_EV_TYPE_NIF && !state->events) { - state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; - state->driver.nif->in.pid = NIL; - state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - state->driver.stop.resource = NULL; - } - ret = INT_MIN | ERL_NIF_SELECT_FAILED; - goto done; + if (on) { + ctl_events &= ~old_events; + state->events |= ctl_events; + state->active_events |= ctl_events; + if (state->type == ERTS_EV_TYPE_NONE) + ctl_op = ERTS_POLL_OP_ADD; + } + else { + ctl_events &= old_events; + state->events &= ~ctl_events; + state->active_events &= ~ctl_events; } - old_events = state->events; + if (ctl_events || ctl_op == ERTS_POLL_OP_DEL) { + ErtsPollEvents new_events; - ASSERT(on - ? (new_events == (state->events | ctl_events)) - : (new_events == (state->events & ~ctl_events))); + new_events = erts_io_control_wakeup(state, ctl_op, + state->active_events, + &wake_poller); + + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + if (state->type == ERTS_EV_TYPE_NIF && !old_events) { + state->type = ERTS_EV_TYPE_NONE; + state->flags = 0; + state->driver.nif->in.pid = NIL; + state->driver.nif->out.pid = NIL; + state->driver.stop.resource = NULL; + } + ret = INT_MIN | ERL_NIF_SELECT_FAILED; + goto done; + } + ASSERT(new_events == state->events); + } ASSERT(state->type == ERTS_EV_TYPE_NIF || state->type == ERTS_EV_TYPE_NONE); - state->events = new_events; if (on) { const Eterm recipient = pid ? pid->pid : env->proc->common.id; Uint32* refn; @@ -1353,7 +1015,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, } ASSERT(state->type == ERTS_EV_TYPE_NIF); ASSERT(state->driver.stop.resource == resource); - if (ctl_events & ERTS_POLL_EV_IN) { + if (mode & ERL_DRV_READ) { state->driver.nif->in.pid = recipient; if (is_immed(ref)) { state->driver.nif->in.immed = ref; @@ -1361,13 +1023,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(is_internal_ref(ref)); refn = internal_ref_numbers(ref); state->driver.nif->in.immed = THE_NON_VALUE; - state->driver.nif->in.refn[0] = refn[0]; - state->driver.nif->in.refn[1] = refn[1]; - state->driver.nif->in.refn[2] = refn[2]; + sys_memcpy(state->driver.nif->in.refn, refn, + sizeof(state->driver.nif->in.refn)); } - state->driver.nif->in.ddeselect_cnt = 0; } - if (ctl_events & ERTS_POLL_EV_OUT) { + if (mode & ERL_DRV_WRITE) { state->driver.nif->out.pid = recipient; if (is_immed(ref)) { state->driver.nif->out.immed = ref; @@ -1375,11 +1035,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(is_internal_ref(ref)); refn = internal_ref_numbers(ref); state->driver.nif->out.immed = THE_NON_VALUE; - state->driver.nif->out.refn[0] = refn[0]; - state->driver.nif->out.refn[1] = refn[1]; - state->driver.nif->out.refn[2] = refn[2]; + sys_memcpy(state->driver.nif->out.refn, refn, + sizeof(state->driver.nif->out.refn)); } - state->driver.nif->out.ddeselect_cnt = 0; } ret = 0; } @@ -1387,14 +1045,9 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, if (state->type == ERTS_EV_TYPE_NIF) { state->driver.nif->in.pid = NIL; state->driver.nif->out.pid = NIL; - state->driver.nif->in.ddeselect_cnt = 0; - state->driver.nif->out.ddeselect_cnt = 0; - if (old_events != 0) { - remember_removed(state, &pollset); - } } - ASSERT(new_events==0); - if (state->remove_cnt == 0 || !wake_poller) { + ASSERT(state->events==0); + if (!wake_poller) { /* * Safe to close fd now as it is not in pollset * or there was no need to eject fd (kernel poll) @@ -1426,14 +1079,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, done: check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif &free_select, &free_nif); done_unknown: - erts_smp_mtx_unlock(fd_mtx(fd)); + erts_mtx_unlock(fd_mtx(fd)); if (call_stop) { erts_resource_stop(resource, (ErlNifEvent)fd, 1); if (call_stop == CALL_STOP_AND_RELEASE) { @@ -1445,162 +1095,9 @@ done_unknown: if (free_nif) free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif return ret; } - -int -ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix, - ErlDrvEvent e, - ErlDrvEventData event_data) -{ -#if !ERTS_CIO_HAVE_DRV_EVENT - return -1; -#else - ErtsSysFdType fd = (ErtsSysFdType) e; - ErtsPollEvents events; - ErtsPollEvents add_events; - ErtsPollEvents remove_events; - Eterm id = erts_drvport2id(ix); - ErtsDrvEventState *state; - int do_wake = 0; - int ret; -#if ERTS_CIO_HAVE_DRV_EVENT - ErtsDrvEventDataState *free_event; -#endif - ErtsDrvSelectDataState *free_select; - ErtsNifSelectDataState *free_nif; - Port *prt = erts_drvport2port(ix); - - if (prt == ERTS_INVALID_ERL_DRV_PORT) - return -1; - - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) { - if (fd < 0) - return -1; - if (fd >= max_fds) { - event_large_fd_error(ix, fd, event_data); - return -1; - } - grow_drv_ev_state(fd); - } -#endif - - erts_smp_mtx_lock(fd_mtx(fd)); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; -#else - /* Could use hash_new directly, but want to keep the normal case fast */ - state = hash_get_drv_ev_state(fd); - if (state == NULL) { - state = hash_new_drv_ev_state(fd); - } -#endif - - switch (state->type) { - case ERTS_EV_TYPE_DRV_EV: - if (state->driver.event->port == id) break; - /*fall through*/ - case ERTS_EV_TYPE_DRV_SEL: - drv_event_steal(ix, state, event_data); - break; - case ERTS_EV_TYPE_STOP_USE: { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, fd, event_data); - steal_pending_stop_use(dsbufp, ix, state, 0, 1); - break; - } - } - - ASSERT(state->type == ERTS_EV_TYPE_DRV_EV - || state->type == ERTS_EV_TYPE_NONE); - - events = state->events; - - if (!event_data) { - remove_events = events; - add_events = 0; - } - else { - remove_events = ~event_data->events & events; - add_events = ~events & event_data->events; - } - - if (add_events) { - events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, add_events, 1, &do_wake); - if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - ret = -1; - goto done; - } - } - if (remove_events) { - events = ERTS_CIO_POLL_CTL(pollset.ps, state->fd, remove_events, 0, &do_wake); - if (events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { - ret = -1; - goto done; - } - } - if (event_data && event_data->events != 0) { - if (state->type == ERTS_EV_TYPE_DRV_EV) { - state->driver.event->removed_events &= ~add_events; - state->driver.event->removed_events |= remove_events; - } - else { - if (!state->driver.event) - state->driver.event = alloc_drv_event_data(); - state->driver.event->port = id; - state->driver.event->removed_events = (ErtsPollEvents) 0; - state->type = ERTS_EV_TYPE_DRV_EV; - } - state->driver.event->data = event_data; - } - else { - if (state->type == ERTS_EV_TYPE_DRV_EV) { - abort_tasks(state, 0); - state->driver.event->port = NIL; - state->driver.event->data = NULL; - state->driver.event->removed_events = (ErtsPollEvents) 0; - } - state->type = ERTS_EV_TYPE_NONE; - remember_removed(state, &pollset); - } - state->events = events; - ASSERT(event_data ? events == event_data->events : events == 0); - - ret = 0; - -done: - - check_fd_cleanup(state, -#if ERTS_CIO_HAVE_DRV_EVENT - &free_event, -#endif - &free_select, - &free_nif); - - erts_smp_mtx_unlock(fd_mtx(fd)); - - if (free_select) - free_drv_select_data(free_select); - if (free_nif) - free_nif_select_data(free_nif); -#if ERTS_CIO_HAVE_DRV_EVENT - if (free_event) - free_drv_event_data(free_event); -#endif - - return ret; -#endif -} - static ERTS_INLINE int chk_stale(Eterm id, ErtsDrvEventState *state, int mode) { @@ -1632,11 +1129,6 @@ need2steal(ErtsDrvEventState *state, int mode) do_steal = 1; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: - do_steal |= chk_stale(state->driver.event->port, state, 0); - break; -#endif case ERTS_EV_TYPE_STOP_USE: case ERTS_EV_TYPE_STOP_NIF: ASSERT(0); @@ -1670,7 +1162,7 @@ print_driver_name(erts_dsprintf_buf_t *dsbufp, Eterm id) static void steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) { - erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "stealing control of fd=%d from ", (int) state->fd); switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: { int deselect_mode = 0; @@ -1694,7 +1186,7 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) if (deselect_mode) deselect(state, deselect_mode); else { - erts_dsprintf(dsbufp, "no one", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "no one", (int) state->fd); ASSERT(0); } erts_dsprintf(dsbufp, "\n"); @@ -1719,30 +1211,13 @@ steal(erts_dsprintf_buf_t *dsbufp, ErtsDrvEventState *state, int mode) erts_dsprintf(dsbufp, "\n"); break; } -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: { - Eterm eid = state->driver.event->port; - if (is_nil(eid)) { - erts_dsprintf(dsbufp, "no one", (int) state->fd); - ASSERT(0); - } - else { - erts_dsprintf(dsbufp, "event driver "); - print_driver_name(dsbufp, eid); - erts_dsprintf(dsbufp, "%T ", eid); - } - erts_dsprintf(dsbufp, "\n"); - deselect(state, 0); - break; - } -#endif case ERTS_EV_TYPE_STOP_USE: case ERTS_EV_TYPE_STOP_NIF: { ASSERT(0); break; } default: - erts_dsprintf(dsbufp, "no one\n", (int) GET_FD(state->fd)); + erts_dsprintf(dsbufp, "no one\n", (int) state->fd); ASSERT(0); } } @@ -1756,7 +1231,7 @@ print_drv_select_op(erts_dsprintf_buf_t *dsbufp, "driver_select(%p, %d,%s%s%s%s, %d) " "by ", ix, - (int) GET_FD(fd), + (int) fd, mode & ERL_DRV_READ ? " ERL_DRV_READ" : "", mode & ERL_DRV_WRITE ? " ERL_DRV_WRITE" : "", mode & ERL_DRV_USE ? " ERL_DRV_USE" : "", @@ -1773,7 +1248,7 @@ print_nif_select_op(erts_dsprintf_buf_t *dsbufp, { erts_dsprintf(dsbufp, "enif_select(_, %d,%s%s%s, %T:%T, %T) ", - (int) GET_FD(fd), + (int) fd, mode & ERL_NIF_SELECT_READ ? " READ" : "", mode & ERL_NIF_SELECT_WRITE ? " WRITE" : "", mode & ERL_NIF_SELECT_STOP ? " STOP" : "", @@ -1812,7 +1287,7 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd) { erts_dsprintf(dsbufp, "fd=%d is larger than the largest allowed fd=%d\n", - (int) fd, max_fds - 1); + (int) fd, drv_ev_state.max_fds - 1); } static void @@ -1868,7 +1343,7 @@ steal_pending_stop_use(erts_dsprintf_buf_t *dsbufp, ErlDrvPort ix, erts_ddll_dereference_driver(state->driver.stop.drv_ptr->handle); } state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; + state->flags = 0; state->driver.stop.drv_ptr = NULL; } else { @@ -1905,9 +1380,9 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource, erts_dsprintf(dsbufp, "called before stop was called for NIF resource %T:%T\n", rt->module, rt->name); - enif_release_resource(state->driver.stop.resource); + enif_release_resource(state->driver.stop.resource->data); state->type = ERTS_EV_TYPE_NONE; - state->flags &= ~ERTS_EV_FLAG_USED; + state->flags = 0; state->driver.stop.resource = NULL; } else { @@ -1917,59 +1392,9 @@ steal_pending_stop_nif(erts_dsprintf_buf_t *dsbufp, ErtsResource* resource, } - -#if ERTS_CIO_HAVE_DRV_EVENT - -static void -print_drv_event_op(erts_dsprintf_buf_t *dsbufp, - ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) -{ - Port *pp = erts_drvport2port(ix); - erts_dsprintf(dsbufp, "driver_event(%p, %d, ", ix, (int) fd); - if (!event_data) - erts_dsprintf(dsbufp, "NULL"); - else - erts_dsprintf(dsbufp, "{0x%x, 0x%x}", - (unsigned int) event_data->events, - (unsigned int) event_data->revents); - erts_dsprintf(dsbufp, ") by "); - if (pp != ERTS_INVALID_ERL_DRV_PORT) - print_driver_name(dsbufp, pp->common.id); - erts_dsprintf(dsbufp, "driver %T ", pp != ERTS_INVALID_ERL_DRV_PORT ? pp->common.id : NIL); -} - -static void -drv_event_steal(ErlDrvPort ix, ErtsDrvEventState *state, ErlDrvEventData event_data) -{ - if (need2steal(state, ERL_DRV_READ|ERL_DRV_WRITE)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, state->fd, event_data); - steal(dsbufp, state, ERL_DRV_READ|ERL_DRV_WRITE); - erts_send_error_to_logger_nogl(dsbufp); - } - else if (state->type == ERTS_EV_TYPE_DRV_SEL) { - ASSERT(state->flags & ERTS_EV_FLAG_USED); - deselect(state, 0); - } -} - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static void -event_large_fd_error(ErlDrvPort ix, ErtsSysFdType fd, ErlDrvEventData event_data) -{ - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - print_drv_event_op(dsbufp, ix, fd, event_data); - erts_dsprintf(dsbufp, "failed: "); - large_fd_error_common(dsbufp, fd); - erts_send_error_to_logger_nogl(dsbufp); -} -#endif -#endif - static ERTS_INLINE int io_task_schedule_allowed(ErtsDrvEventState *state, - ErtsPortTaskType type, - erts_aint_t current_cio_time) + ErtsPortTaskType type) { ErtsIoTask *io_task; @@ -1977,71 +1402,53 @@ io_task_schedule_allowed(ErtsDrvEventState *state, case ERTS_PORT_TASK_INPUT: if (!state->driver.select) return 0; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - return 0; -#endif io_task = &state->driver.select->iniotask; break; case ERTS_PORT_TASK_OUTPUT: if (!state->driver.select) return 0; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - return 0; -#endif io_task = &state->driver.select->outiotask; break; -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_PORT_TASK_EVENT: - if (!state->driver.event) - return 0; - if (state->driver.select) - return 0; - io_task = &state->driver.event->iotask; - break; -#endif default: ERTS_INTERNAL_ERROR("Invalid I/O-task type"); return 0; } - return !is_iotask_active(io_task, current_cio_time); + return !is_iotask_active(io_task); } static ERTS_INLINE void -iready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) +iready(Eterm id, ErtsDrvEventState *state) { if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_INPUT, - current_cio_time)) { + ERTS_PORT_TASK_INPUT)) { ErtsIoTask *iotask = &state->driver.select->iniotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_INPUT, (ErlDrvEvent) state->fd) != 0) { stale_drv_select(id, state, ERL_DRV_READ); - } - add_active_fd(state->fd); + } else { + DEBUG_PRINT_FD("schedule ready_input(%T, %d)", + state, id, state->fd); + } } } static ERTS_INLINE void -oready(Eterm id, ErtsDrvEventState *state, erts_aint_t current_cio_time) +oready(Eterm id, ErtsDrvEventState *state) { if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_OUTPUT, - current_cio_time)) { + ERTS_PORT_TASK_OUTPUT)) { ErtsIoTask *iotask = &state->driver.select->outiotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); if (erts_port_task_schedule(id, &iotask->task, ERTS_PORT_TASK_OUTPUT, (ErlDrvEvent) state->fd) != 0) { stale_drv_select(id, state, ERL_DRV_WRITE); - } - add_active_fd(state->fd); + } else { + DEBUG_PRINT_FD("schedule ready_output(%T, %d)", state, id, state->fd); + } } } @@ -2090,141 +1497,62 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, erts_queue_message(rp, rp_locks, mp, tuple, am_system); if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); + erts_proc_unlock(rp, rp_locks); } - -#if ERTS_CIO_HAVE_DRV_EVENT -static ERTS_INLINE void -eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data, - erts_aint_t current_cio_time) -{ - if (io_task_schedule_allowed(state, - ERTS_PORT_TASK_EVENT, - current_cio_time)) { - ErtsIoTask *iotask = &state->driver.event->iotask; - erts_smp_atomic_set_nob(&iotask->executed_time, current_cio_time); - if (erts_port_task_schedule(id, - &iotask->task, - ERTS_PORT_TASK_EVENT, - (ErlDrvEvent) state->fd, - event_data) != 0) { - stale_drv_select(id, state, 0); - } - add_active_fd(state->fd); - } -} -#endif - static void bad_fd_in_pollset(ErtsDrvEventState *, Eterm inport, Eterm outport); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT void -ERTS_CIO_EXPORT(erts_check_io_async_sig_interrupt)(void) +erts_check_io_interrupt(ErtsPollThread *psi, int set) { - ERTS_CIO_POLL_AS_INTR(pollset.ps); -} + if (psi) { +#if ERTS_POLL_USE_FALLBACK + if (psi->ps == get_fallback()) { + erts_poll_interrupt_flbk(psi->ps, set); + return; + } #endif - -void -ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set) -{ - ERTS_CIO_POLL_INTR(pollset.ps, set); + erts_poll_interrupt(psi->ps, set); + } } -void -ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set, - ErtsMonotonicTime timeout_time) -{ - ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, timeout_time); +ErtsPollThread * +erts_create_pollset_thread(int id) { + return psiv+id; } -#if !ERTS_CIO_DEFER_ACTIVE_EVENTS -/* - * Number of ignored events, for a lingering fd added by enif_select(), - * until we deselect fd-event from pollset. - */ -# define ERTS_NIF_DELAYED_DESELECT 20 -#else -/* Disable delayed deselect as pollset cannot handle active events */ -# define ERTS_NIF_DELAYED_DESELECT 1 -#endif - void -ERTS_CIO_EXPORT(erts_check_io)(int do_wait) +erts_check_io(ErtsPollThread *psi) { - ErtsPollResFd *pollres; int pollres_len; - ErtsMonotonicTime timeout_time; int poll_ret, i; - erts_aint_t current_cio_time; - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - - ASSERT(esdp); + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_CHECK_IO); restart: -#ifdef ERTS_BREAK_REQUESTED - if (ERTS_BREAK_REQUESTED) - erts_do_break_handling(); -#endif - -#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ - if (ERTS_SIGNAL_STATE) { - erts_handle_signal_state(); - } -#endif - - /* Figure out timeout value */ - timeout_time = (do_wait - ? erts_check_next_timeout_time(esdp) - : ERTS_POLL_NO_TIMEOUT /* poll only */); - - /* - * No need for an atomic inc op when incrementing - * erts_check_io_time, since only one thread can - * check io at a time. - */ - current_cio_time = erts_smp_atomic_read_dirty(&erts_check_io_time); - current_cio_time++; - erts_smp_atomic_set_relb(&erts_check_io_time, current_cio_time); - - check_cleanup_active_fds(current_cio_time, - timeout_time != ERTS_POLL_NO_TIMEOUT); - #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - pollres_len = erts_smp_atomic32_read_dirty(&pollset.active_fd.no) + ERTS_CHECK_IO_POLL_RES_LEN; + pollres_len = psi->pollres_len; - pollres = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollResFd)*pollres_len); +#if ERTS_POLL_USE_FALLBACK + if (psi->ps == get_fallback()) { - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1); + poll_ret = erts_poll_wait_flbk(psi->ps, psi->pollres, &pollres_len); - poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, timeout_time); + } else +#endif + { + poll_ret = erts_poll_wait(psi->ps, psi->pollres, &pollres_len); + } #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif -#ifdef ERTS_BREAK_REQUESTED - if (ERTS_BREAK_REQUESTED) - erts_do_break_handling(); -#endif - - -#ifdef ERTS_SIGNAL_STATE /* ifndef ERTS_SMP */ - if (ERTS_SIGNAL_STATE) { - erts_handle_signal_state(); - } -#endif - - if (poll_ret != 0) { - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); - forget_removed(&pollset); - erts_free(ERTS_ALC_T_TMP, pollres); + if (poll_ret == EAGAIN) { goto restart; } @@ -2240,65 +1568,86 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erl_errno_id(poll_ret), poll_ret); erts_send_error_to_logger_nogl(dsbufp); } + ERTS_MSACC_POP_STATE(); return; } for (i = 0; i < pollres_len; i++) { - ErtsSysFdType fd = (ErtsSysFdType) pollres[i].fd; + erts_driver_t* drv_ptr = NULL; + ErtsResource* resource = NULL; + ErtsDrvSelectDataState *free_select = NULL; + ErtsNifSelectDataState *free_nif = NULL; + ErtsSysFdType fd = (ErtsSysFdType) ERTS_POLL_RES_GET_FD(&psi->pollres[i]); ErtsDrvEventState *state; + ErtsPollEvents revents; - erts_smp_mtx_lock(fd_mtx(fd)); + erts_mtx_lock(fd_mtx(fd)); + + state = get_drv_ev_state(fd); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[ (int) fd]; -#else - state = hash_get_drv_ev_state(fd); if (!state) { - goto next_pollres; + erts_mtx_unlock(fd_mtx(fd)); + continue; } -#endif - /* Skip this fd if it was removed from pollset */ - if (is_removed(state)) { - goto next_pollres; - } + revents = ERTS_POLL_RES_GET_EVTS(&psi->pollres[i]); + + DEBUG_PRINT_FD("triggered %s", state, ev2str(revents)); + + if (revents & ERTS_POLL_EV_ERR) { + /* + * Handle error events by triggering all in/out events + * that has been selected on. + * We *do not* want to call a callback that corresponds + * to an event not selected. + */ + revents = state->active_events; + state->active_events = 0; + } else { + + /* Disregard any events that are not active at the moment, + for instance this could happen if the driver/nif does + select/deselect in rapid succession. */ + revents &= state->active_events | ERTS_POLL_EV_NVAL; + state->active_events &= ~revents; + + /* Reactivate the poll op if there are still active events */ + if (state->active_events) { + ErtsPollEvents new_events; + DEBUG_PRINT_FD("re-enable %s", state, ev2str(state->active_events)); + + new_events = erts_io_control(state, ERTS_POLL_OP_MOD, state->active_events); + + /* Unable to re-enable the fd, signal all callbacks */ + if (new_events & (ERTS_POLL_EV_ERR|ERTS_POLL_EV_NVAL)) { + revents |= state->active_events; + state->active_events = 0; + } + } + } switch (state->type) { case ERTS_EV_TYPE_DRV_SEL: { /* Requested via driver_select()... */ - ErtsPollEvents revents = pollres[i].events; - - if (revents & ERTS_POLL_EV_ERR) { - /* - * Handle error events by triggering all in/out events - * that the driver has selected. - * We *do not* want to call a callback that corresponds - * to an event not selected. - */ - revents = state->events; - } - else { - revents &= (state->events | ERTS_POLL_EV_NVAL); - } if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { - oready(state->driver.select->outport, state, current_cio_time); + oready(state->driver.select->outport, state); } /* Someone might have deselected input since revents was read (true also on the non-smp emulator since oready() may have been called); therefore, update revents... */ - revents &= state->events; + revents &= state->events; if (revents & ERTS_POLL_EV_IN) { - iready(state->driver.select->inport, state, current_cio_time); + iready(state->driver.select->inport, state); } } else if (revents & ERTS_POLL_EV_NVAL) { bad_fd_in_pollset(state, state->driver.select->inport, state->driver.select->outport); - add_active_fd(state->fd); + check_fd_cleanup(state, &free_select, &free_nif); } break; } @@ -2307,112 +1656,116 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) struct erts_nif_select_event in = {NIL}; struct erts_nif_select_event out = {NIL}; ErtsResource* resource = NULL; - ErtsPollEvents revents = pollres[i].events; - - if (revents & ERTS_POLL_EV_ERR) { - /* - * Handle error events by triggering all in/out events - * that the NIF has selected. - * We *do not* want to send a message that corresponds - * to an event not selected. - */ - revents = state->events; - } - else { - revents &= (state->events | ERTS_POLL_EV_NVAL); - } if (revents & (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) { if (revents & ERTS_POLL_EV_OUT) { if (is_not_nil(state->driver.nif->out.pid)) { out = state->driver.nif->out; resource = state->driver.stop.resource; - state->driver.nif->out.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; state->driver.nif->out.pid = NIL; - add_active_fd(state->fd); - } - else { - ASSERT(state->driver.nif->out.ddeselect_cnt >= 2); - state->driver.nif->out.ddeselect_cnt--; } } if (revents & ERTS_POLL_EV_IN) { if (is_not_nil(state->driver.nif->in.pid)) { in = state->driver.nif->in; resource = state->driver.stop.resource; - state->driver.nif->in.ddeselect_cnt = ERTS_NIF_DELAYED_DESELECT; state->driver.nif->in.pid = NIL; - add_active_fd(state->fd); - } - else { - ASSERT(state->driver.nif->in.ddeselect_cnt >= 2); - state->driver.nif->in.ddeselect_cnt--; } } + state->events &= ~revents; } else if (revents & ERTS_POLL_EV_NVAL) { bad_fd_in_pollset(state, NIL, NIL); - add_active_fd(state->fd); + check_fd_cleanup(state, &free_select, &free_nif); } -#ifdef ERTS_SMP - erts_smp_mtx_unlock(fd_mtx(fd)); -#endif + erts_mtx_unlock(fd_mtx(fd)); + if (is_not_nil(in.pid)) { send_event_tuple(&in, resource, am_ready_input); } if (is_not_nil(out.pid)) { send_event_tuple(&out, resource, am_ready_output); } - goto next_pollres_unlocked; + continue; } -#if ERTS_CIO_HAVE_DRV_EVENT - case ERTS_EV_TYPE_DRV_EV: { /* Requested via driver_event()... */ - ErlDrvEventData event_data; - ErtsPollEvents revents; - ASSERT(state->driver.event); - ASSERT(state->driver.event->data); - event_data = state->driver.event->data; - revents = pollres[i].events; - revents &= ~state->driver.event->removed_events; - - if (revents) { - event_data->events = state->events; - event_data->revents = revents; - eready(state->driver.event->port, state, event_data, current_cio_time); - } - break; - } -#endif + case ERTS_EV_TYPE_STOP_NIF: { + resource = state->driver.stop.resource; + state->type = ERTS_EV_TYPE_NONE; + goto case_ERTS_EV_TYPE_NONE; + } + case ERTS_EV_TYPE_STOP_USE: { +#if ERTS_POLL_USE_FALLBACK + ASSERT(psi->ps == get_fallback()); +#endif + drv_ptr = state->driver.stop.drv_ptr; + state->type = ERTS_EV_TYPE_NONE; + /* fallthrough */ case ERTS_EV_TYPE_NONE: /* Deselected ... */ + case_ERTS_EV_TYPE_NONE: + ASSERT(!state->events && !state->active_events && !state->flags); + check_fd_cleanup(state, &free_select, &free_nif); break; + } default: { /* Error */ erts_dsprintf_buf_t *dsbufp; dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Invalid event request type for fd in erts_poll()! " - "fd=%d, event request type=%sd\n", (int) state->fd, + "fd=%d, event request type=%d\n", (int) state->fd, (int) state->type); ASSERT(0); deselect(state, 0); - add_active_fd(state->fd); break; } } - next_pollres:; -#ifdef ERTS_SMP - erts_smp_mtx_unlock(fd_mtx(fd)); -#endif - next_pollres_unlocked:; - } + erts_mtx_unlock(fd_mtx(fd)); - erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0); - erts_free(ERTS_ALC_T_TMP, pollres); - forget_removed(&pollset); + if (drv_ptr) { + int was_unmasked = erts_block_fpe(); + DTRACE1(driver_stop_select, drv_ptr->name); + LTTNG1(driver_stop_select, drv_ptr->name); + (*drv_ptr->stop_select)((ErlDrvEvent) fd, NULL); + erts_unblock_fpe(was_unmasked); + if (drv_ptr->handle) { + erts_ddll_dereference_driver(drv_ptr->handle); + } + } + if (resource) { + erts_resource_stop(resource, (ErlNifEvent)fd, 1); + enif_release_resource(resource->data); + } + if (free_select) + free_drv_select_data(free_select); + if (free_nif) + free_nif_select_data(free_nif); + } + + /* The entire pollres array was filled with events, + * grow it for the next call. We do this for two reasons: + * 1. Pulling out more events in on go will increase throughput + * 2. If the polling implementation is not fair, this will make + * sure that we get all fds that we can. i.e. if 12 fds are + * constantly active, but we only have a pollres_len of 10, + * two of the fds may never be triggered depending on what the + * kernel decides to do. + **/ + if (pollres_len == psi->pollres_len) { + int ev_state_len = drv_ev_state_len(); + erts_free(ERTS_ALC_T_POLLSET, psi->pollres); + psi->pollres_len *= 2; + /* Never grow it larger than the current drv_ev_state.len size */ + if (psi->pollres_len > ev_state_len) + psi->pollres_len = ev_state_len; + psi->pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * psi->pollres_len); + } + + ERTS_MSACC_POP_STATE(); } static void @@ -2506,16 +1859,16 @@ static int drv_ev_state_cmp(void *des1, void *des2) static void *drv_ev_state_alloc(void *des_tmpl) { ErtsDrvEventState *evstate; - erts_smp_spin_lock(&state_prealloc_lock); - if (state_prealloc_first == NULL) { - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + if (drv_ev_state.prealloc_first == NULL) { + erts_spin_unlock(&drv_ev_state.prealloc_lock); evstate = (ErtsDrvEventState *) erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)); } else { - evstate = state_prealloc_first; - state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next; - --num_state_prealloc; - erts_smp_spin_unlock(&state_prealloc_lock); + evstate = drv_ev_state.prealloc_first; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) evstate->hb.next; + --drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } /* XXX: Already valid data if prealloced, could ignore template! */ *evstate = *((ErtsDrvEventState *) des_tmpl); @@ -2525,63 +1878,201 @@ static void *drv_ev_state_alloc(void *des_tmpl) static void drv_ev_state_free(void *des) { - erts_smp_spin_lock(&state_prealloc_lock); - ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb; - state_prealloc_first = (ErtsDrvEventState *) des; - ++num_state_prealloc; - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + ((ErtsDrvEventState *) des)->hb.next = &drv_ev_state.prealloc_first->hb; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) des; + ++drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } #endif +#define ERTS_MAX_NO_OF_POLL_THREADS ERTS_MAX_NO_OF_SCHEDULERS + +static char * +get_arg(char* rest, char** argv, int* ip) +{ + int i = *ip; + if (*rest == '\0') { + if (argv[i+1] == NULL) { + erts_fprintf(stderr, "too few arguments\n"); + erts_usage(); + } + argv[i++] = NULL; + rest = argv[i]; + } + argv[i] = NULL; + *ip = i; + return rest; +} + +static void +parse_args(int *argc, char **argv, int concurrent_waiters) +{ + int i = 0, j; + int no_pollsets = 0, no_poll_threads = 0, + no_pollsets_percentage = 0, + no_poll_threads_percentage = 0; + ASSERT(argc && argv); + while (i < *argc) { + if(argv[i][0] == '-') { + switch (argv[i][1]) { + case 'I': { + if (strncmp(argv[i]+2, "Ot", 2) == 0) { + char *arg = get_arg(argv[i]+4, argv, &i); + if (sscanf(arg, "%d", &no_poll_threads) != 1 || + no_poll_threads < 1 || + ERTS_MAX_NO_OF_POLL_THREADS < no_poll_threads) { + erts_fprintf(stderr,"bad I/O poll threads number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "Op", 3) == 0) { + char *arg = get_arg(argv[i]+4, argv, &i); + if (sscanf(arg, "%d", &no_pollsets) != 1 || + no_pollsets < 1) { + erts_fprintf(stderr,"bad I/O pollset number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "OPt", 4) == 0) { + char *arg = get_arg(argv[i]+5, argv, &i); + if (sscanf(arg, "%d", &no_poll_threads_percentage) != 1 || + no_poll_threads_percentage < 0 || + no_poll_threads_percentage > 100) { + erts_fprintf(stderr,"bad I/O poll thread percentage number: %s\n", arg); + erts_usage(); + } + } else if (strncmp(argv[i]+2, "OPp", 4) == 0) { + char *arg = get_arg(argv[i]+5, argv, &i); + if (sscanf(arg, "%d", &no_pollsets_percentage) != 1 || + no_pollsets_percentage < 0 || + no_pollsets_percentage > 100) { + erts_fprintf(stderr,"bad I/O pollset percentage number: %s\n", arg); + erts_usage(); + } + } else { + break; + } + break; + } + case 'K': + (void)get_arg(argv[i]+2, argv, &i); + break; + case '-': + goto args_parsed; + default: + break; + } + } + i++; + } + +args_parsed: + + if (!concurrent_waiters) { + no_pollsets = no_poll_threads; + no_pollsets_percentage = 100; + } + + if (no_poll_threads == 0) { + if (no_poll_threads_percentage == 0) + no_poll_threads = 1; /* This is the default */ + else { + no_poll_threads = erts_no_schedulers * no_poll_threads_percentage / 100; + if (no_poll_threads < 1) + no_poll_threads = 1; + } + } + + if (no_pollsets == 0) { + if (no_pollsets_percentage == 0) + no_pollsets = 1; /* This is the default */ + else { + no_pollsets = no_poll_threads * no_pollsets_percentage / 100; + if (no_pollsets < 1) + no_pollsets = 1; + } + } + + if (no_poll_threads < no_pollsets) { + erts_fprintf(stderr, + "number of IO poll threads has to be greater or equal to " + "the number of \nIO pollsets. Current values are set to: \n" + " -IOt %d -IOp %d\n", + no_poll_threads, no_pollsets); + erts_usage(); + } + + /* Handled arguments have been marked with NULL. Slide arguments + not handled towards the beginning of argv. */ + for (i = 0, j = 0; i < *argc; i++) { + if (argv[i]) + argv[j++] = argv[i]; + } + *argc = j; + + erts_no_pollsets = no_pollsets; + erts_no_poll_threads = no_poll_threads; +} + void -ERTS_CIO_EXPORT(erts_init_check_io)(void) +erts_init_check_io(int *argc, char **argv) { + int j, concurrent_waiters, no_poll_threads; ERTS_CT_ASSERT((INT_MIN & (ERL_NIF_SELECT_STOP_CALLED | ERL_NIF_SELECT_STOP_SCHEDULED | ERL_NIF_SELECT_INVALID_EVENT | ERL_NIF_SELECT_FAILED)) == 0); - erts_smp_atomic_init_nob(&erts_check_io_time, 0); - erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0); - - ERTS_CIO_POLL_INIT(); - pollset.ps = ERTS_CIO_NEW_POLLSET(); - - pollset.active_fd.six = 0; - pollset.active_fd.eix = 0; - erts_smp_atomic32_init_nob(&pollset.active_fd.no, 0); - pollset.active_fd.size = ERTS_ACTIVE_FD_INC; - pollset.active_fd.array = erts_alloc(ERTS_ALC_T_ACTIVE_FD_ARR, - sizeof(ErtsSysFdType)*ERTS_ACTIVE_FD_INC); -#ifdef DEBUG - { - int i; - for (i = 0; i < ERTS_ACTIVE_FD_INC; i++) - pollset.active_fd.array[i] = ERTS_SYS_FD_INVALID; - } + + + erts_poll_init(&concurrent_waiters); +#if ERTS_POLL_USE_FALLBACK + erts_poll_init_flbk(NULL); #endif + parse_args(argc, argv, concurrent_waiters); -#ifdef ERTS_SMP - init_removed_fd_alloc(); - pollset.removed_list = NULL; - erts_smp_spinlock_init(&pollset.removed_list_lock, - "pollset_rm_list"); - { - int i; - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i)); -#else - erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state"); + /* Create the actual pollsets */ + pollsetv = erts_alloc(ERTS_ALC_T_POLLSET,sizeof(ErtsPollSet *) * erts_no_pollsets); + + for (j=0; j < erts_no_pollsets; j++) + pollsetv[j] = erts_poll_create_pollset(j); + +#if ERTS_POLL_USE_FALLBACK + flbk_pollset = erts_poll_create_pollset_flbk(-1); #endif - } - } + + no_poll_threads = erts_no_poll_threads; +#if ERTS_POLL_USE_FALLBACK + no_poll_threads++; +#endif + + psiv = erts_alloc(ERTS_ALC_T_POLLSET, sizeof(ErtsPollThread) * no_poll_threads); + +#if ERTS_POLL_USE_FALLBACK + psiv[0].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN; + psiv[0].pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN); + psiv[0].ps = get_fallback(); + psiv++; #endif + + for (j = 0; j < erts_no_poll_threads; j++) { + psiv[j].pollres_len = ERTS_CHECK_IO_POLL_RES_LEN; + psiv[j].pollres = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(ErtsPollResFd) * ERTS_CHECK_IO_POLL_RES_LEN); + psiv[j].ps = pollsetv[j % erts_no_pollsets]; + } + + for (j=0; j < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; j++) { + erts_mtx_init(&drv_ev_state.locks[j].lck, "drv_ev_state", make_small(j), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + } + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_smp_atomic_init_nob(&drv_ev_state_len, 0); - drv_ev_state = NULL; - erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow"); + drv_ev_state.max_fds = erts_poll_max_fds(); + erts_atomic_init_nob(&drv_ev_state.len, 0); + drv_ev_state.v = NULL; + erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); #else { SafeHashFunctions hf; @@ -2589,152 +2080,182 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) hf.cmp = &drv_ev_state_cmp; hf.alloc = &drv_ev_state_alloc; hf.free = &drv_ev_state_free; - num_state_prealloc = 0; - state_prealloc_first = NULL; - erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc"); - - safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", - DRV_EV_STATE_HTAB_SIZE, hf); + drv_ev_state.num_prealloc = 0; + drv_ev_state.prealloc_first = NULL; + erts_spinlock_init(&drv_ev_state.prealloc_lock, "state_prealloc", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state.tab, "drv_ev_state_tab", + ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf); } #endif } int -ERTS_CIO_EXPORT(erts_check_io_max_files)(void) +erts_check_io_max_files(void) { #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - return max_fds; + return drv_ev_state.max_fds; #else - return ERTS_POLL_EXPORT(erts_poll_max_fds)(); + return erts_poll_max_fds(); #endif } Uint -ERTS_CIO_EXPORT(erts_check_io_size)(void) +erts_check_io_size(void) { - Uint res; + Uint res = 0; ErtsPollInfo pi; - ERTS_CIO_POLL_INFO(pollset.ps, &pi); - res = pi.memory_size; + int i; + +#if ERTS_POLL_USE_FALLBACK + erts_poll_info(get_fallback(), &pi); + res += pi.memory_size; +#endif + + for (i = 0; i < erts_no_pollsets; i++) { + erts_poll_info(pollsetv[i], &pi); + res += pi.memory_size; + } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - res += safe_hash_table_sz(&drv_ev_state_tab); + res += safe_hash_table_sz(&drv_ev_state.tab); { SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); + safe_hash_get_info(&hi, &drv_ev_state.tab); res += hi.objs * sizeof(ErtsDrvEventState); } - erts_smp_spin_lock(&state_prealloc_lock); - res += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + res += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif return res; } Eterm -ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) +erts_check_io_info(void *proc) { Process *p = (Process *) proc; - Eterm tags[16], values[16], res; - Uint sz, *szp, *hp, **hpp, memory_size; - Sint i; - ErtsPollInfo pi; - erts_aint_t cio_time = erts_smp_atomic_read_acqb(&erts_check_io_time); - int active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); + Eterm tags[16], values[16], res, list = NIL; + Uint sz, *szp, *hp, **hpp; + ErtsPollInfo *piv; + Sint i, j = 0, len; + int no_pollsets = erts_no_pollsets + ERTS_POLL_USE_FALLBACK; + ERTS_CT_ASSERT(ERTS_POLL_USE_FALLBACK == 0 || ERTS_POLL_USE_FALLBACK == 1); - while (1) { - erts_aint_t post_cio_time; - int post_active_fds; + piv = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollInfo) * no_pollsets); - ERTS_CIO_POLL_INFO(pollset.ps, &pi); +#if ERTS_POLL_USE_FALLBACK + erts_poll_info_flbk(get_fallback(), &piv[0]); + piv[0].poll_threads = 1; + piv[0].active_fds = 0; + piv++; +#endif - post_cio_time = erts_smp_atomic_read_mb(&erts_check_io_time); - post_active_fds = (int) erts_smp_atomic32_read_acqb(&pollset.active_fd.no); - if (cio_time == post_cio_time && active_fds == post_active_fds) - break; - cio_time = post_cio_time; - active_fds = post_active_fds; + for (j = 0; j < erts_no_pollsets; j++) { + erts_poll_info(pollsetv[j], &piv[j]); + piv[j].active_fds = 0; + piv[j].poll_threads = erts_no_poll_threads / erts_no_pollsets; + if (erts_no_poll_threads % erts_no_pollsets > j) + piv[j].poll_threads++; } - memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len); + i = 0; + erts_mtx_lock(&drv_ev_state.grow_lock); + len = erts_atomic_read_nob(&drv_ev_state.len); + for (i = 0; i < ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_lock(&drv_ev_state.locks[i].lck); + for (j = i; j < len; j+=ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT) { + ErtsDrvEventState *state = get_drv_ev_state(j); + int pollsetid = get_pollset_id(j); + ASSERT(fd_mtx(j) == &drv_ev_state.locks[i].lck); + if (state->flags & ERTS_EV_FLAG_FALLBACK) + pollsetid = -1; + if (state->driver.select + && (state->type == ERTS_EV_TYPE_DRV_SEL) + && (is_iotask_active(&state->driver.select->iniotask) + || is_iotask_active(&state->driver.select->outiotask))) + piv[pollsetid].active_fds++; + } + erts_mtx_unlock(&drv_ev_state.locks[i].lck); + } + erts_mtx_unlock(&drv_ev_state.grow_lock); + + piv[0].memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - memory_size += safe_hash_table_sz(&drv_ev_state_tab); + piv[0].memory_size += safe_hash_table_sz(&drv_ev_state.tab); { - SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); - memory_size += hi.objs * sizeof(ErtsDrvEventState); + SafeHashInfo hi; + safe_hash_get_info(&hi, &drv_ev_state.tab); + piv[0].memory_size += hi.objs * sizeof(ErtsDrvEventState); } - erts_smp_spin_lock(&state_prealloc_lock); - memory_size += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_smp_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + piv[0].memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif hpp = NULL; szp = &sz; sz = 0; - bld_it: - i = 0; + piv -= ERTS_POLL_USE_FALLBACK; - tags[i] = erts_bld_atom(hpp, szp, "name"); - values[i++] = erts_bld_atom(hpp, szp, "erts_poll"); + bld_it: - tags[i] = erts_bld_atom(hpp, szp, "primary"); - values[i++] = erts_bld_atom(hpp, szp, pi.primary); + for (j = no_pollsets-1; j >= 0; j--) { + i = 0; - tags[i] = erts_bld_atom(hpp, szp, "fallback"); - values[i++] = erts_bld_atom(hpp, szp, pi.fallback ? pi.fallback : "false"); + tags[i] = erts_bld_atom(hpp, szp, "name"); + values[i++] = erts_bld_atom(hpp, szp, "erts_poll"); - tags[i] = erts_bld_atom(hpp, szp, "kernel_poll"); - values[i++] = erts_bld_atom(hpp, szp, - pi.kernel_poll ? pi.kernel_poll : "false"); + tags[i] = erts_bld_atom(hpp, szp, "primary"); + values[i++] = erts_bld_atom(hpp, szp, piv[j].primary); - tags[i] = erts_bld_atom(hpp, szp, "memory_size"); - values[i++] = erts_bld_uint(hpp, szp, memory_size); + tags[i] = erts_bld_atom(hpp, szp, "kernel_poll"); + values[i++] = erts_bld_atom(hpp, szp, + piv[j].kernel_poll ? piv[j].kernel_poll : "false"); - tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.poll_set_size); + tags[i] = erts_bld_atom(hpp, szp, "memory_size"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].memory_size); - if (pi.fallback) { - tags[i] = erts_bld_atom(hpp, szp, "fallback_poll_set_size"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.fallback_poll_set_size); - } + tags[i] = erts_bld_atom(hpp, szp, "total_poll_set_size"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_set_size); - tags[i] = erts_bld_atom(hpp, szp, "lazy_updates"); - values[i++] = pi.lazy_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "lazy_updates"); + values[i++] = piv[j].lazy_updates ? am_true : am_false; - if (pi.lazy_updates) { - tags[i] = erts_bld_atom(hpp, szp, "pending_updates"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.pending_updates); - } + tags[i] = erts_bld_atom(hpp, szp, "pending_updates"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].pending_updates); - tags[i] = erts_bld_atom(hpp, szp, "batch_updates"); - values[i++] = pi.batch_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "batch_updates"); + values[i++] = piv[j].batch_updates ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates"); - values[i++] = pi.concurrent_updates ? am_true : am_false; + tags[i] = erts_bld_atom(hpp, szp, "concurrent_updates"); + values[i++] = piv[j].concurrent_updates ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "max_fds"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.max_fds); + tags[i] = erts_bld_atom(hpp, szp, "fallback"); + values[i++] = piv[j].is_fallback ? am_true : am_false; - tags[i] = erts_bld_atom(hpp, szp, "active_fds"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) active_fds); + tags[i] = erts_bld_atom(hpp, szp, "max_fds"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].max_fds); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - tags[i] = erts_bld_atom(hpp, szp, "no_avoided_wakeups"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_wakeups); + tags[i] = erts_bld_atom(hpp, szp, "active_fds"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].active_fds); - tags[i] = erts_bld_atom(hpp, szp, "no_avoided_interrupts"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_avoided_interrupts); + tags[i] = erts_bld_atom(hpp, szp, "poll_threads"); + values[i++] = erts_bld_uint(hpp, szp, piv[j].poll_threads); - tags[i] = erts_bld_atom(hpp, szp, "no_interrupt_timed"); - values[i++] = erts_bld_uint(hpp, szp, (Uint) pi.no_interrupt_timed); -#endif + res = erts_bld_2tup_list(hpp, szp, i, tags, values); - res = erts_bld_2tup_list(hpp, szp, i, tags, values); + if (!hpp) { + *szp += 2; + } + else { + list = CONS(*hpp, res, list); + *hpp += 2; + } + } if (!hpp) { hp = HAlloc(p, sz); @@ -2743,386 +2264,442 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) goto bld_it; } - return res; + erts_free(ERTS_ALC_T_TMP, piv); + + return list; } static ERTS_INLINE ErtsPollEvents -print_events(ErtsPollEvents ev) +print_events(erts_dsprintf_buf_t *dsbufp, ErtsPollEvents ev) { int first = 1; + if(ev == ERTS_POLL_EV_NONE) { + erts_dsprintf(dsbufp, "N/A"); + return 0; + } if(ev & ERTS_POLL_EV_IN) { ev &= ~ERTS_POLL_EV_IN; - erts_printf("%s%s", first ? "" : "|", "IN"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "IN"); first = 0; } if(ev & ERTS_POLL_EV_OUT) { ev &= ~ERTS_POLL_EV_OUT; - erts_printf("%s%s", first ? "" : "|", "OUT"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "OUT"); first = 0; } /* The following should not appear... */ if(ev & ERTS_POLL_EV_NVAL) { - erts_printf("%s%s", first ? "" : "|", "NVAL"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "NVAL"); first = 0; } if(ev & ERTS_POLL_EV_ERR) { - erts_printf("%s%s", first ? "" : "|", "ERR"); + erts_dsprintf(dsbufp, "%s%s", first ? "" : "|", "ERR"); first = 0; } if (ev) - erts_printf("%s0x%b32x", first ? "" : "|", (Uint32) ev); + erts_dsprintf(dsbufp, "%s0x%b32x", first ? "" : "|", (Uint32) ev); return ev; } +static ERTS_INLINE void +print_flags(erts_dsprintf_buf_t *dsbufp, EventStateFlags f) +{ + const char* delim = ""; + if(f & ERTS_EV_FLAG_USED) { + erts_dsprintf(dsbufp, "%s","USED"); + delim = "|"; + } + if(f & ERTS_EV_FLAG_FALLBACK) { + erts_dsprintf(dsbufp, "%s%s", delim, "FLBK"); + delim = "|"; + } +} + +#ifdef DEBUG_PRINT_MODE + +static ERTS_INLINE char * +drvmode2str(int mode) { + switch (mode) { + case ERL_DRV_READ|ERL_DRV_USE: return "READ|USE"; + case ERL_DRV_WRITE|ERL_DRV_USE: return "WRITE|USE"; + case ERL_DRV_READ|ERL_DRV_WRITE|ERL_DRV_USE: return "READ|WRITE|USE"; + case ERL_DRV_USE: return "USE"; + case ERL_DRV_READ: return "READ"; + case ERL_DRV_WRITE: return "WRITE"; + case ERL_DRV_READ|ERL_DRV_WRITE: return "READ|WRITE"; + default: return "UNKNOWN"; + } +} + +static ERTS_INLINE char * +nifmode2str(enum ErlNifSelectFlags mode) { + switch (mode) { + case ERL_NIF_SELECT_READ: return "READ"; + case ERL_NIF_SELECT_WRITE: return "WRITE"; + case ERL_NIF_SELECT_STOP: return "STOP"; + default: return "UNKNOWN"; + } +} + +#endif + typedef struct { int used_fds; int num_errors; int no_driver_select_structs; - int no_driver_event_structs; + int no_enif_select_structs; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS int internal_fds; ErtsPollEvents *epep; #endif } IterDebugCounters; -static void doit_erts_check_io_debug(void *vstate, void *vcounters) +static int erts_debug_print_checkio_state(erts_dsprintf_buf_t *dsbufp, + ErtsDrvEventState *state, + ErtsPollEvents ep_events, + int internal) { - ErtsDrvEventState *state = (ErtsDrvEventState *) vstate; - IterDebugCounters *counters = (IterDebugCounters *) vcounters; - ErtsPollEvents cio_events = state->events; - ErtsSysFdType fd = state->fd; -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - int internal = 0; - ErtsPollEvents ep_events = counters->epep[(int) fd]; -#endif - int err = 0; - #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE) struct stat stat_buf; #endif - - if (state->driver.select) - counters->no_driver_select_structs++; -#if ERTS_CIO_HAVE_DRV_EVENT - if (state->driver.event) - counters->no_driver_event_structs++; -#endif - + ErtsSysFdType fd = state->fd; + ErtsPollEvents cio_events = state->events; + int err = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (state->events || ep_events) { - if (ep_events & ERTS_POLL_EV_NVAL) { - ep_events &= ~ERTS_POLL_EV_NVAL; - internal = 1; - counters->internal_fds++; - } - else - counters->used_fds++; -#else - if (state->events) { - counters->used_fds++; + ErtsPollEvents aio_events = state->active_events; #endif - - erts_printf("fd=%d ", (int) fd); - + erts_dsprintf(dsbufp, "pollset=%d fd=%d ", + state->flags & ERTS_EV_FLAG_FALLBACK ? -1 : get_pollset_id(fd), (int) fd); + #if defined(HAVE_FSTAT) && !defined(NO_FSTAT_ON_SYS_FD_TYPE) - if (fstat((int) fd, &stat_buf) < 0) - erts_printf("type=unknown "); - else { - erts_printf("type="); + if (fstat((int) fd, &stat_buf) < 0) + erts_dsprintf(dsbufp, "type=unknown "); + else { + erts_dsprintf(dsbufp, "type="); #ifdef S_ISSOCK - if (S_ISSOCK(stat_buf.st_mode)) - erts_printf("sock "); - else + if (S_ISSOCK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "sock "); + else #endif #ifdef S_ISFIFO if (S_ISFIFO(stat_buf.st_mode)) - erts_printf("fifo "); + erts_dsprintf(dsbufp, "fifo "); else #endif #ifdef S_ISCHR - if (S_ISCHR(stat_buf.st_mode)) - erts_printf("chr "); - else + if (S_ISCHR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "chr "); + else #endif #ifdef S_ISDIR - if (S_ISDIR(stat_buf.st_mode)) - erts_printf("dir "); - else + if (S_ISDIR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "dir "); + else #endif #ifdef S_ISBLK - if (S_ISBLK(stat_buf.st_mode)) - erts_printf("blk "); - else + if (S_ISBLK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "blk "); + else #endif #ifdef S_ISREG - if (S_ISREG(stat_buf.st_mode)) - erts_printf("reg "); - else + if (S_ISREG(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "reg "); + else #endif #ifdef S_ISLNK - if (S_ISLNK(stat_buf.st_mode)) - erts_printf("lnk "); - else + if (S_ISLNK(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "lnk "); + else #endif #ifdef S_ISDOOR - if (S_ISDOOR(stat_buf.st_mode)) - erts_printf("door "); - else + if (S_ISDOOR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "door "); + else #endif #ifdef S_ISWHT - if (S_ISWHT(stat_buf.st_mode)) - erts_printf("wht "); - else + if (S_ISWHT(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "wht "); + else #endif #ifdef S_ISXATTR - if (S_ISXATTR(stat_buf.st_mode)) - erts_printf("xattr "); - else + if (S_ISXATTR(stat_buf.st_mode)) + erts_dsprintf(dsbufp, "xattr "); + else #endif - erts_printf("unknown "); - } + erts_dsprintf(dsbufp, "unknown "); + } #else - erts_printf("type=unknown "); + erts_dsprintf(dsbufp, "type=unknown "); #endif - if (state->type == ERTS_EV_TYPE_DRV_SEL) { - erts_printf("driver_select "); - -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - - if (cio_events == ep_events) { - erts_printf("ev="); - if (print_events(cio_events) != 0) - err = 1; - } - else { - err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - erts_printf(" ep_ev="); - print_events(ep_events); - } -#else - if (print_events(cio_events) != 0) - err = 1; -#endif - erts_printf(" "); - if (cio_events & ERTS_POLL_EV_IN) { - Eterm id = state->driver.select->inport; - if (is_nil(id)) { - erts_printf("inport=none inname=none indrv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" inport=%T inname=%s indrv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } - if (cio_events & ERTS_POLL_EV_OUT) { - Eterm id = state->driver.select->outport; - if (is_nil(id)) { - erts_printf("outport=none outname=none outdrv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" outport=%T outname=%s outdrv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } - } - else if (state->type == ERTS_EV_TYPE_NIF) { - ErtsResource* r; - erts_printf("enif_select "); + if (state->type == ERTS_EV_TYPE_DRV_SEL) { + erts_dsprintf(dsbufp, "driver_select "); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - + if (internal) { + erts_dsprintf(dsbufp, "internal "); + err = 1; + } + if (aio_events == cio_events) { if (cio_events == ep_events) { - erts_printf("ev="); - if (print_events(cio_events) != 0) + erts_dsprintf(dsbufp, "ev="); + if (print_events(dsbufp, cio_events) != 0) err = 1; } else { + ErtsPollEvents ev = cio_events; + if (ev != ep_events && ep_events != ERTS_POLL_EV_NONE) + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); + } + } else { + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " aio_ev="); + print_events(dsbufp, aio_events); + if ((aio_events != ep_events && ep_events != ERTS_POLL_EV_NONE) || + (aio_events != 0 && ep_events == ERTS_POLL_EV_NONE)) { + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - erts_printf(" ep_ev="); - print_events(ep_events); } + } #else - if (print_events(cio_events) != 0) + if (print_events(dsbufp, cio_events) != 0) + err = 1; +#endif + erts_dsprintf(dsbufp, " "); + if (cio_events & ERTS_POLL_EV_IN) { + Eterm id = state->driver.select->inport; + if (is_nil(id)) { + erts_dsprintf(dsbufp, "inport=none inname=none indrv=none "); err = 1; -#endif - erts_printf(" inpid=%T dd_cnt=%b32d", state->driver.nif->in.pid, - state->driver.nif->in.ddeselect_cnt); - erts_printf(" outpid=%T dd_cnt=%b32d", state->driver.nif->out.pid, - state->driver.nif->out.ddeselect_cnt); - r = state->driver.stop.resource; - erts_printf(" resource=%p(%T:%T)", r, r->type->module, r->type->name); + } + else { + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, " inport=%T inname=%s indrv=%s ", + id, + pnp->name ? pnp->name : "unknown", + (pnp->driver_name + ? pnp->driver_name + : "unknown")); + erts_free_port_names(pnp); + } + } + if (cio_events & ERTS_POLL_EV_OUT) { + Eterm id = state->driver.select->outport; + if (is_nil(id)) { + erts_dsprintf(dsbufp, "outport=none outname=none outdrv=none "); + err = 1; + } + else { + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); + erts_dsprintf(dsbufp, " outport=%T outname=%s outdrv=%s ", + id, + pnp->name ? pnp->name : "unknown", + (pnp->driver_name + ? pnp->driver_name + : "unknown")); + erts_free_port_names(pnp); + } } -#if ERTS_CIO_HAVE_DRV_EVENT - else if (state->type == ERTS_EV_TYPE_DRV_EV) { - Eterm id; - erts_printf("driver_event "); + } + else if (state->type == ERTS_EV_TYPE_NIF) { + ErtsResource* r; + erts_dsprintf(dsbufp, "enif_select "); + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (internal) { - erts_printf("internal "); - err = 1; - } - if (cio_events == ep_events) { - erts_printf("ev=0x%b32x", (Uint32) cio_events); - } - else { - err = 1; - erts_printf("cio_ev=0x%b32x", (Uint32) cio_events); - erts_printf(" ep_ev=0x%b32x", (Uint32) ep_events); - } + if (internal) { + erts_dsprintf(dsbufp, "internal "); + err = 1; + } + + if (cio_events == ep_events) { + erts_dsprintf(dsbufp, "ev="); + if (print_events(dsbufp, cio_events) != 0) + err = 1; + } + else { + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); + print_events(dsbufp, ep_events); + } #else - erts_printf("ev=0x%b32x", (Uint32) cio_events); + if (print_events(dsbufp, cio_events) != 0) + err = 1; #endif - id = state->driver.event->port; - if (is_nil(id)) { - erts_printf(" port=none name=none drv=none "); - err = 1; - } - else { - ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); - erts_printf(" port=%T name=%s drv=%s ", - id, - pnp->name ? pnp->name : "unknown", - (pnp->driver_name - ? pnp->driver_name - : "unknown")); - erts_free_port_names(pnp); - } - } + erts_dsprintf(dsbufp, " inpid=%T", state->driver.nif->in.pid); + erts_dsprintf(dsbufp, " outpid=%T", state->driver.nif->out.pid); + r = state->driver.stop.resource; + erts_dsprintf(dsbufp, " resource=%p(%T:%T)", r, r->type->module, r->type->name); + } +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + else if (internal) { + erts_dsprintf(dsbufp, "internal "); + if (cio_events) { + err = 1; + erts_dsprintf(dsbufp, "cio_ev="); + print_events(dsbufp, cio_events); + } + if (ep_events) { + erts_dsprintf(dsbufp, "ep_ev="); + print_events(dsbufp, ep_events); + } + } #endif + else { + err = 1; + erts_dsprintf(dsbufp, "control_type=%d ", (int)state->type); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - else if (internal) { - erts_printf("internal "); - if (cio_events) { - err = 1; - erts_printf("cio_ev="); - print_events(cio_events); - } - if (ep_events) { - erts_printf("ep_ev="); - print_events(ep_events); - } - } + if (cio_events == ep_events) { + erts_dsprintf(dsbufp, "ev="); + print_events(dsbufp, cio_events); + } + else { + erts_dsprintf(dsbufp, "cio_ev="); print_events(dsbufp, cio_events); + erts_dsprintf(dsbufp, " ep_ev="); print_events(dsbufp, ep_events); + } +#else + erts_dsprintf(dsbufp, "ev=0x%b32x", (Uint32) cio_events); #endif - else { - err = 1; - erts_printf("control_type=%d ", (int)state->type); + } + + erts_dsprintf(dsbufp, " flags="); print_flags(dsbufp, state->flags); + if (err) { + erts_dsprintf(dsbufp, " ERROR"); + } + erts_dsprintf(dsbufp, "\r\n"); + return err; +} + +static void doit_erts_check_io_debug(void *vstate, void *vcounters, + erts_dsprintf_buf_t *dsbufp) +{ + ErtsDrvEventState *state = (ErtsDrvEventState *) vstate; + IterDebugCounters *counters = (IterDebugCounters *) vcounters; + int internal = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if (cio_events == ep_events) { - erts_printf("ev="); - print_events(cio_events); - } - else { - erts_printf("cio_ev="); print_events(cio_events); - erts_printf(" ep_ev="); print_events(ep_events); - } + ErtsSysFdType fd = state->fd; + ErtsPollEvents ep_events = counters->epep[(int) fd]; #else - erts_printf("ev=0x%b32x", (Uint32) cio_events); + ErtsPollEvents ep_events = ERTS_POLL_EV_NONE; #endif + + if (state->driver.select) { + counters->no_driver_select_structs++; + ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)); + } + if (state->driver.nif) { + counters->no_enif_select_structs++; + ASSERT(state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)); + } + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + if (state->events || (ep_events != 0 && ep_events != ERTS_POLL_EV_NONE)) { + if (ep_events & ERTS_POLL_EV_NVAL) { + ep_events &= ~ERTS_POLL_EV_NVAL; + internal = 1; + counters->internal_fds++; } - - if (err) { + else + counters->used_fds++; +#else + if (state->events) { + counters->used_fds++; +#endif + if (erts_debug_print_checkio_state(dsbufp, state, ep_events, internal)) { counters->num_errors++; - erts_printf(" ERROR"); } - erts_printf("\n"); } } - + +/* ciodpi can be NULL when called from etp-commands */ int -ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) +erts_check_io_debug(ErtsCheckIoDebugInfo *ciodip) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - int fd, len; + int fd, len, i; #endif - IterDebugCounters counters; + IterDebugCounters counters = {0}; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS ErtsDrvEventState null_des; null_des.driver.select = NULL; -#if ERTS_CIO_HAVE_DRV_EVENT - null_des.driver.event = NULL; -#endif + null_des.driver.nif = NULL; null_des.driver.stop.drv_ptr = NULL; null_des.events = 0; - null_des.remove_cnt = 0; null_des.type = ERTS_EV_TYPE_NONE; + null_des.flags = 0; + + counters.epep = erts_alloc(ERTS_ALC_T_TMP, + sizeof(ErtsPollEvents)*drv_ev_state.max_fds); #endif - erts_printf("--- fds in pollset --------------------------------------\n"); -#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +#if defined(ERTS_ENABLE_LOCK_CHECK) erts_lc_check_exact(NULL, 0); /* No locks should be locked */ #endif - erts_smp_thr_progress_block(); /* stop the world to avoid messy locking */ + if (ciodip) + erts_thr_progress_block(); /* stop the world to avoid messy locking */ #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds); - ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds); - counters.internal_fds = 0; -#endif - counters.used_fds = 0; - counters.num_errors = 0; - counters.no_driver_select_structs = 0; - counters.no_driver_event_structs = 0; + len = erts_atomic_read_nob(&drv_ev_state.len); -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_smp_atomic_read_nob(&drv_ev_state_len); +#if ERTS_POLL_USE_FALLBACK + erts_dsprintf(dsbufp, "--- fds in flbk pollset ---------------------------------\n"); + erts_poll_get_selected_events_flbk(get_fallback(), counters.epep, + drv_ev_state.max_fds); for (fd = 0; fd < len; fd++) { - doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); + if (drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK) + doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp); + } +#endif + erts_dsprintf(dsbufp, "--- fds in pollset --------------------------------------\n"); + + for (i = 0; i < erts_no_pollsets; i++) { + erts_poll_get_selected_events(pollsetv[i], + counters.epep, + drv_ev_state.max_fds); + for (fd = 0; fd < len; fd++) { + if (!(drv_ev_state.v[fd].flags & ERTS_EV_FLAG_FALLBACK) + && get_pollset_id(fd) == i) + doit_erts_check_io_debug(&drv_ev_state.v[fd], &counters, dsbufp); + } } - for ( ; fd < max_fds; fd++) { - null_des.fd = fd; - doit_erts_check_io_debug((void *) &null_des, (void *) &counters); + for (fd = len ; fd < drv_ev_state.max_fds; fd++) { + null_des.fd = fd; + doit_erts_check_io_debug(&null_des, &counters, dsbufp); } #else - safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters); + safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug, + &counters, dsbufp); #endif - erts_smp_thr_progress_unblock(); + if (ciodip) + erts_thr_progress_unblock(); - ciodip->no_used_fds = counters.used_fds; - ciodip->no_driver_select_structs = counters.no_driver_select_structs; - ciodip->no_driver_event_structs = counters.no_driver_event_structs; + if (ciodip) { + ciodip->no_used_fds = counters.used_fds; + ciodip->no_driver_select_structs = counters.no_driver_select_structs; + ciodip->no_enif_select_structs = counters.no_enif_select_structs; + } - erts_printf("\n"); - erts_printf("used fds=%d\n", counters.used_fds); - erts_printf("Number of driver_select() structures=%d\n", counters.no_driver_select_structs); -#if ERTS_CIO_HAVE_DRV_EVENT - erts_printf("Number of driver_event() structures=%d\n", counters.no_driver_event_structs); -#endif + erts_dsprintf(dsbufp, "\n"); + erts_dsprintf(dsbufp, "used fds=%d\n", counters.used_fds); + erts_dsprintf(dsbufp, "Number of driver_select() structures=%d\n", counters.no_driver_select_structs); + erts_dsprintf(dsbufp, "Number of enif_select() structures=%d\n", counters.no_enif_select_structs); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - erts_printf("internal fds=%d\n", counters.internal_fds); + erts_dsprintf(dsbufp, "internal fds=%d\n", counters.internal_fds); #endif - erts_printf("---------------------------------------------------------\n"); - fflush(stdout); + erts_dsprintf(dsbufp, "---------------------------------------------------------\n"); + erts_send_error_to_logger_nogl(dsbufp); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS erts_free(ERTS_ALC_T_TMP, (void *) counters.epep); #endif @@ -3130,3 +2707,20 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) return counters.num_errors; } +#ifdef ERTS_ENABLE_LOCK_COUNT +void erts_lcnt_update_cio_locks(int enable) { + int i; +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS + erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); +#else + (void)enable; +#endif + +#if ERTS_POLL_USE_FALLBACK + erts_lcnt_enable_pollset_lock_count_flbk(get_fallback(), enable); +#endif + + for (i = 0; i < erts_no_pollsets; i++) + erts_lcnt_enable_pollset_lock_count(pollsetv[i], enable); +} +#endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h index ee4abeece9..443ef1264c 100644 --- a/erts/emulator/sys/common/erl_check_io.h +++ b/erts/emulator/sys/common/erl_check_io.h @@ -18,10 +18,11 @@ * %CopyrightEnd% */ -/* - * Description: Check I/O +/** + * @description Check I/O, a cross platform IO polling framework for ERTS * - * Author: Rickard Green + * @author Rickard Green + * @author Lukas Larsson */ #ifndef ERL_CHECK_IO_H__ @@ -30,70 +31,79 @@ #include "sys.h" #include "erl_sys_driver.h" -#ifdef ERTS_ENABLE_KERNEL_POLL - -int driver_select_kp(ErlDrvPort, ErlDrvEvent, int, int); -int driver_select_nkp(ErlDrvPort, ErlDrvEvent, int, int); -int enif_select_kp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); -int enif_select_nkp(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); -int driver_event_kp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); -int driver_event_nkp(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); -Uint erts_check_io_size_kp(void); -Uint erts_check_io_size_nkp(void); -Eterm erts_check_io_info_kp(void *); -Eterm erts_check_io_info_nkp(void *); -int erts_check_io_max_files_kp(void); -int erts_check_io_max_files_nkp(void); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void erts_check_io_async_sig_interrupt_kp(void); -void erts_check_io_async_sig_interrupt_nkp(void); -#endif -void erts_check_io_interrupt_kp(int); -void erts_check_io_interrupt_nkp(int); -void erts_check_io_interrupt_timed_kp(int, ErtsMonotonicTime); -void erts_check_io_interrupt_timed_nkp(int, ErtsMonotonicTime); -void erts_check_io_kp(int); -void erts_check_io_nkp(int); -void erts_init_check_io_kp(void); -void erts_init_check_io_nkp(void); -int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *); -int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *); - -#else /* !ERTS_ENABLE_KERNEL_POLL */ +/** @brief a structure that is used by each polling thread */ +struct erts_poll_thread; +/** + * Get the memory size of the check io framework + */ Uint erts_check_io_size(void); -Eterm erts_check_io_info(void *); +/** + * Returns an Eterm with information about all the pollsets active at the + * moment. + * + * @param proc the Process* to allocate the result on. It is passed as + * void * because of header include problems. + */ +Eterm erts_check_io_info(void *proc); +/** + * Should be called when a port IO task has been executed in order to re-enable + * or clear the information about the fd. + * + * @param type The type of event that has been completed. + * @param handle The port task handle of the event. + * @param reset A function pointer to be called when the port task handle + * should be reset. + */ +void erts_io_notify_port_task_executed(ErtsPortTaskType type, + ErtsPortTaskHandle *handle, + void (*reset)(ErtsPortTaskHandle *)); +/** + * Returns the maximum number of fds that the check io framework can handle. + */ int erts_check_io_max_files(void); -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void erts_check_io_async_sig_interrupt(void); -#endif -void erts_check_io_interrupt(int); -void erts_check_io_interrupt_timed(int, ErtsMonotonicTime); -void erts_check_io(int); -void erts_init_check_io(void); - +/** + * Called by any thread that should check for new IO events. This function will + * not return unless erts_check_io_interrupt(pt, 1) is called by another thread. + * + * @param pt the poll thread structure to use. + */ +void erts_check_io(struct erts_poll_thread *pt); +/** + * Initialize the check io framework. This function will parse the arguments + * and delete any entries that it is interested in. + * + * @param argc the number of arguments + * @param argv an array with the arguments + */ +void erts_init_check_io(int *argc, char **argv); +/** + * Interrupt the poll thread so that it can execute other code. + * + * Should be called with set = 0 by the waiting thread before calling + * erts_check_io. + * + * @param pt the poll thread to wake + * @param set whether to set or clear the interrupt flag + */ +void erts_check_io_interrupt(struct erts_poll_thread *pt, int set); +/** + * Create a new poll thread structure that is associated with the number no. + * It is the callers responsibility that no is unique. + */ +struct erts_poll_thread* erts_create_pollset_thread(int no); +#ifdef ERTS_ENABLE_LOCK_COUNT +/** + * Toggle lock counting on all check io locks + */ +void erts_lcnt_update_cio_locks(int enable); #endif -extern erts_smp_atomic_t erts_check_io_time; - typedef struct { ErtsPortTaskHandle task; - erts_smp_atomic_t executed_time; + ErtsSysFdType fd; } ErtsIoTask; -ERTS_GLB_INLINE void erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE void -erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) -{ - ErtsIoTask *itp = (ErtsIoTask *) (((char *) pthp) - offsetof(ErtsIoTask, task)); - erts_aint_t ci_time = erts_smp_atomic_read_acqb(&erts_check_io_time); - erts_smp_atomic_set_relb(&itp->executed_time, ci_time); -} - -#endif #endif /* ERL_CHECK_IO_H__ */ @@ -101,6 +111,16 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) #define ERL_CHECK_IO_INTERNAL__ #endif +#define ERTS_CHECK_IO_DRV_EV_STATE_LOCK_CNT 128 + +/* Controls how many pollsets to allocate. Fd's are hashed into + each pollset based on the FD. When doing non-concurrent updates + there will be one pollset per thread. +*/ +extern int erts_no_pollsets; +extern int erts_no_poll_threads; + + #ifndef ERL_CHECK_IO_INTERNAL__ #define ERL_CHECK_IO_INTERNAL__ #include "erl_poll.h" @@ -113,24 +133,8 @@ erts_io_notify_port_task_executed(ErtsPortTaskHandle *pthp) */ # define ERTS_CIO_DEFER_ACTIVE_EVENTS 1 #else -# define ERTS_CIO_DEFER_ACTIVE_EVENTS 0 -#endif - -/* - * ErtsDrvEventDataState is used by driver_event() which is almost never - * used. We allocate ErtsDrvEventDataState separate since we dont wan't - * the size of ErtsDrvEventState to increase due to driver_event() - * information. - */ -typedef struct { - Eterm port; - ErlDrvEventData data; - ErtsPollEvents removed_events; -#if ERTS_CIO_DEFER_ACTIVE_EVENTS - ErtsPollEvents deferred_events; +# define ERTS_CIO_DEFER_ACTIVE_EVENTS 1 #endif - ErtsIoTask iotask; -} ErtsDrvEventDataState; typedef struct { Eterm inport; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index bb930ff03b..a9c6e72c5f 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -24,7 +24,6 @@ #define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" -#include "erl_smp.h" #include "atom.h" #include "erl_mmap.h" #include <stddef.h> @@ -62,11 +61,11 @@ (((UWord) (PTR)) - ((UWord) mm->sa.bot) \ < ((UWord) mm->sua.top) - ((UWord) mm->sa.bot)) #define ERTS_MMAP_IN_SUPERALIGNED_AREA(PTR) \ - (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ + (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ (((UWord) (PTR)) - ((UWord) mm->sa.bot) \ < ((UWord) mm->sa.top) - ((UWord) mm->sa.bot))) #define ERTS_MMAP_IN_SUPERUNALIGNED_AREA(PTR) \ - (ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ + (ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mm->mtx)), \ (((UWord) (PTR)) - ((UWord) mm->sua.bot) \ < ((UWord) mm->sua.top) - ((UWord) mm->sua.bot))) @@ -199,10 +198,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MMAP_OP_LCK(RES, IN_SZ, OUT_SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MMAP_OP_START((IN_SZ)); \ ERTS_MMAP_OP_END((RES), (OUT_SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MUNMAP_OP(PTR, SZ) \ @@ -221,9 +220,9 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MUNMAP_OP_LCK(PTR, SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MUNMAP_OP((PTR), (SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MREMAP_OP_START(OLD_PTR, OLD_SZ, IN_SZ) \ @@ -249,10 +248,10 @@ static ErtsMMapOp mmap_ops[ERTS_MMAP_OP_RINGBUF_SZ]; #define ERTS_MREMAP_OP_LCK(RES, OLD_PTR, OLD_SZ, IN_SZ, OUT_SZ) \ do { \ - erts_smp_mtx_lock(&mm->mtx); \ + erts_mtx_lock(&mm->mtx); \ ERTS_MREMAP_OP_START((OLD_PTR), (OLD_SZ), (IN_SZ)); \ ERTS_MREMAP_OP_END((RES), (OUT_SZ)); \ - erts_smp_mtx_unlock(&mm->mtx); \ + erts_mtx_unlock(&mm->mtx); \ } while (0) #define ERTS_MMAP_OP_ABORT() \ @@ -321,7 +320,7 @@ struct ErtsMemMapper_ { #if HAVE_MMAP && (!defined(MAP_ANON) && !defined(MAP_ANONYMOUS)) int mmap_fd; #endif - erts_smp_mtx_t mtx; + erts_mtx_t mtx; struct { char *free_list; char *unused_start; @@ -1536,7 +1535,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) ErtsFreeSegDesc *desc; Uint32 superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); ERTS_MMAP_OP_START(*sizep); @@ -1660,7 +1659,7 @@ erts_mmap(ErtsMemMapper* mm, Uint32 flags, UWord *sizep) } ERTS_MMAP_OP_ABORT(); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } #if ERTS_HAVE_OS_MMAP @@ -1724,13 +1723,13 @@ supercarrier_success: #endif ERTS_MMAP_OP_END(seg, asize); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = asize; return (void *) seg; supercarrier_reserve_failure: - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = 0; return NULL; } @@ -1760,7 +1759,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size) start = (char *) ptr; end = start + size; - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); ERTS_MUNMAP_OP(ptr, size); @@ -1829,7 +1828,7 @@ erts_munmap(ErtsMemMapper* mm, Uint32 flags, void *ptr, UWord size) if (unres_sz) mm->unreserve_physical(((char *) ptr) + ad_sz, unres_sz); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } } } @@ -1948,12 +1947,12 @@ erts_mremap(ErtsMemMapper* mm, ? ERTS_SUPERALIGNED_CEILING(*sizep) : ERTS_PAGEALIGNED_CEILING(*sizep)); - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); if (ERTS_MMAP_IN_SUPERALIGNED_AREA(ptr) ? (!superaligned && lookup_free_seg(&mm->sua.map, asize)) : (superaligned && lookup_free_seg(&mm->sa.map, asize))) { - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); /* * Segment currently in wrong area (due to a previous memory * shortage), move it to the right area. @@ -2068,7 +2067,7 @@ erts_mremap(ErtsMemMapper* mm, } ERTS_MMAP_OP_ABORT(); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); /* Failed to resize... */ } @@ -2090,14 +2089,14 @@ supercarrier_resize_success: #endif ERTS_MREMAP_OP_END(new_ptr, asize); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = asize; return new_ptr; supercarrier_reserve_failure: ERTS_MREMAP_OP_END(NULL, old_size); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); *sizep = old_size; return NULL; @@ -2212,9 +2211,11 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable) erts_exit(1, "erts_mmap: Failed to open /dev/zero\n"); #endif - erts_smp_mtx_init(&mm->mtx, "erts_mmap"); + erts_mtx_init(&mm->mtx, "erts_mmap", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); if (is_first_call) { - erts_mtx_init(&am.init_mutex, "mmap_init_atoms"); + erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION @@ -2405,7 +2406,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm, Eterm res = THE_NON_VALUE; if (!hpp) { - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); emis->sizes[0] = mm->size.supercarrier.total; emis->sizes[1] = mm->sa.top - mm->sa.bot; emis->sizes[2] = mm->sua.top - mm->sua.bot; @@ -2421,7 +2422,7 @@ Eterm erts_mmap_info(ErtsMemMapper* mm, emis->segs[5] = mm->sua.map.nseg; emis->os_used = mm->size.os.used; - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); } list[lix] = erts_mmap_info_options(mm, "option ", print_to_p, print_to_arg, @@ -2541,14 +2542,14 @@ Eterm erts_mmap_debug_info(Process* p) Eterm *hp, *hp_end; Uint may_need; - erts_smp_mtx_lock(&mm->mtx); + erts_mtx_lock(&mm->mtx); values[0] = (UWord)mm->sa.bot; values[1] = (UWord)mm->sa.top; values[2] = (UWord)mm->sua.bot; values[3] = (UWord)mm->sua.top; sa_list = build_free_seg_list(p, &mm->sa.map); sua_list = build_free_seg_list(p, &mm->sua.map); - erts_smp_mtx_unlock(&mm->mtx); + erts_mtx_unlock(&mm->mtx); may_need = 4*(2+3+2) + 2*(2+3); hp = HAlloc(p, may_need); @@ -2816,7 +2817,8 @@ static void hard_dbg_mseg_init(void) { ErtsFreeSegDesc_fake* p; - erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg"); + erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); hard_dbg_mseg_tree.root = NULL; hard_dbg_mseg_tree.order = ADDR_ORDER; diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index b8f0bb7150..bf6de9b13a 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -188,7 +188,6 @@ typedef union { static int no_mseg_allocators; static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr; -#ifdef ERTS_SMP #define ERTS_MSEG_ALLCTR_IX(IX) \ (&aligned_mseg_allctr[(IX)].mseg_alloc) @@ -199,18 +198,6 @@ static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr; #define ERTS_MSEG_ALLCTR_OPT(OPT) \ ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0)) -#else - -#define ERTS_MSEG_ALLCTR_IX(IX) \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#define ERTS_MSEG_ALLCTR_SS() \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#define ERTS_MSEG_ALLCTR_OPT(OPT) \ - (&aligned_mseg_allctr[0].mseg_alloc) - -#endif #define ERTS_MSEG_LOCK(MA) \ do { \ @@ -352,11 +339,11 @@ mseg_recreate(ErtsMsegAllctr_t *ma, Uint flags, void *old_seg, UWord old_size, U do { \ if ((MA)->is_thread_safe) \ ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&(MA)->mtx) \ - || erts_smp_thr_progress_is_blocking() \ + || erts_thr_progress_is_blocking() \ || ERTS_IS_CRASH_DUMPING); \ else \ ERTS_LC_ASSERT((MA)->ix == (int) erts_get_scheduler_id() \ - || erts_smp_thr_progress_is_blocking() \ + || erts_thr_progress_is_blocking() \ || ERTS_IS_CRASH_DUMPING); \ } while (0) #else @@ -1404,11 +1391,7 @@ erts_mseg_init(ErtsMsegInit_t *init) int i; UWord x; -#ifdef ERTS_SMP no_mseg_allocators = init->nos + 1; -#else - no_mseg_allocators = 1; -#endif x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t) *no_mseg_allocators @@ -1420,7 +1403,8 @@ erts_mseg_init(ErtsMsegInit_t *init) atoms_initialized = 0; - erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); + erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); #ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability @@ -1449,7 +1433,8 @@ erts_mseg_init(ErtsMsegInit_t *init) ma->is_thread_safe = 0; else { ma->is_thread_safe = 1; - erts_mtx_init(&ma->mtx, "mseg"); + erts_mtx_init(&ma->mtx, "mseg", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR); } ma->is_cache_check_scheduled = 0; diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c index d53190fdd5..341845cc2a 100644 --- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c +++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c @@ -23,7 +23,6 @@ #endif #include "erl_os_monotonic_time_extender.h" -#ifdef USE_THREADS static void *os_monotonic_time_extender(void *vstatep) { @@ -49,30 +48,22 @@ static void *os_monotonic_time_extender(void *vstatep) } static erts_tid_t os_monotonic_extender_tid; -#endif void erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep, Uint32 (*raw_os_monotonic_time)(void), int check_seconds) { -#ifdef USE_THREADS statep->raw_os_monotonic_time = raw_os_monotonic_time; erts_atomic32_init_nob(&statep->extend[0], (erts_aint32_t) 0); erts_atomic32_init_nob(&statep->extend[1], (erts_aint32_t) 0); statep->check_interval = check_seconds; -#else - statep->extend[0] = (Uint32) 0; - statep->extend[1] = (Uint32) 0; - statep->last_msb = (ErtsMonotonicTime) 0; -#endif } void erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep) { -#ifdef USE_THREADS erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; thr_opts.detached = 1; thr_opts.suggested_stack_size = 4; @@ -85,5 +76,4 @@ erts_late_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep os_monotonic_time_extender, (void*) statep, &thr_opts); -#endif } diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h index 8089c9aed9..53c32579d5 100644 --- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.h +++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.h @@ -25,36 +25,16 @@ #include "erl_threads.h" typedef struct { -#ifdef USE_THREADS Uint32 (*raw_os_monotonic_time)(void); erts_atomic32_t extend[2]; int check_interval; -#else - Uint32 extend[2]; - ErtsMonotonicTime last_msb; -#endif } ErtsOsMonotonicTimeExtendState; -#ifdef USE_THREADS -# define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) ((void) 1) # define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT) \ ((((ErtsMonotonicTime) \ erts_atomic32_read_nob(&((S)->extend[((int) ((RT) >> 31)) & 1]))) \ << 32) \ + (RT)) -#else -# define ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(S, RT) \ - do { \ - Uint32 msb__ = (RT) & (((Uint32) 1) << 31); \ - if (msb__ != (S)->last_msb) { \ - int ix__ = ((int) ((S)->last_msb >> 31)) & 1; \ - (S)->extend[ix__]++; \ - (S)->last_msb = msb; \ - } \ - } while (0) -# define ERTS_EXTEND_OS_MONOTONIC_TIME(S, RT) \ - ((((ErtsMonotonicTime) (S)->extend[((int) ((RT) >> 31)) & 1]) << 32) + (RT)) -#endif void erts_init_os_monotonic_time_extender(ErtsOsMonotonicTimeExtendState *statep, diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 5e7ae8953a..30a595c17a 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -18,9 +18,8 @@ * %CopyrightEnd% */ -/* - * Description: Poll interface suitable for ERTS with or without - * SMP support. +/** + * @description Poll interface suitable for ERTS * * The interface is currently implemented using: * - select @@ -29,12 +28,36 @@ * - epoll with poll or select as fallback * - kqueue with poll or select as fallback * - * Some time in the future it will also be - * implemented using Solaris ports. * + * @author Rickard Green + * @author Lukas Larsson + * + * There are two major different implementations off IO polling in this + * file. The concurrent and non-concurrent implementations. + * When available epoll/kqueue are used to implement the concurrent + * versions. poll, select and dev/poll use non-concurrent updates. + * + * Concurrent version: + * In the concurrent version erts_poll_control directly modifies + * the kernel pollset without waking the thread that is waiting + * on events. Also the ErtsPollResFd type is directly mapped to + * the native event type, so no extra copying is needed. Note that + * as no locking at all is done, fds can be triggered that have been + * removed from the pollset. The check_io layer has to deal with this. + * + * Non-concurrent version: + * In the non-concurrent version, the pollset has an internal representation + * of the pollset that is updated by erts_poll_control. When an fd is updated, + * its number is placed in the update request queue and then the waiting thread + * is woken in order to see the change. The internal data in the pollset is + * protected by a mutex that has to be taken by both the modifying and waiting + * thread at different times. * + * The non-concurrent pollset cannot have fd's closed in it while a thread is + * waiting on that fd. In order to fix this, when an ERTS_POLL_OP_DEL command + * is issued, the fd is marked as closing and the waiting thread is woken. The + * fd is then returned in the waiting threads results as ERTS_POLL_EV_NONE. * - * Author: Rickard Green */ #ifdef HAVE_CONFIG_H @@ -62,6 +85,8 @@ # ifdef SYS_SELECT_H # include <sys/select.h> # endif +#elif defined(_DARWIN_UNLIMITED_SELECT) +# undef _DARWIN_UNLIMITED_SELECT #endif #ifdef NO_SYSCONF # if ERTS_POLL_USE_SELECT @@ -83,23 +108,35 @@ #error "Missing implementation of erts_poll()" #endif -#if defined(ERTS_KERNEL_POLL_VERSION) && !ERTS_POLL_USE_KERNEL_POLL -#error "Missing kernel poll implementation of erts_poll()" -#endif +#if 0 +#define ERTS_POLL_DEBUG_PRINT 1 -#if defined(ERTS_NO_KERNEL_POLL_VERSION) && ERTS_POLL_USE_KERNEL_POLL -#error "Kernel poll used when it shouldn't be used" -#endif +#define DEBUG_PRINT(FMT, PS, ...) \ + do { \ + int myerrno = errno; \ + erts_printf("%d: " FMT "\r\n", (PS)->id, ##__VA_ARGS__); \ + errno = myerrno; \ + } while(0) -#if 0 -#define ERTS_POLL_DEBUG_PRINT +/* Define to print info about modifications done to each fd */ +#define DEBUG_PRINT_FD(FMT, PS, FD, ...) DEBUG_PRINT("%d: " FMT, PS, FD, ##__VA_ARGS__) +/* Define to print entry and exit from erts_poll_wait (can be very spammy) */ +//#define DEBUG_PRINT_WAIT(FMT, PS, ...) DEBUG_PRINT(FMT, PS, ##__VA_ARGS__) + +#else +#define ERTS_POLL_DEBUG_PRINT 0 +#define DEBUG_PRINT(...) #endif -#if defined(DEBUG) && 0 -#define HARD_DEBUG +#ifndef DEBUG_PRINT_FD +#define DEBUG_PRINT_FD(...) +#endif +#ifndef DEBUG_PRINT_WAIT +#define DEBUG_PRINT_WAIT(...) #endif -#ifdef _DARWIN_UNLIMITED_SELECT + +#if defined(_DARWIN_UNLIMITED_SELECT) && ERTS_POLL_USE_SELECT typedef struct { size_t sz; fd_set* ptr; @@ -145,74 +182,43 @@ int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, # define ERTS_SELECT select #endif -#define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \ - || ERTS_POLL_USE_KQUEUE) -#define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE \ - (defined(ERTS_SMP) || ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_POLL) - -#define ERTS_POLL_USE_CONCURRENT_UPDATE \ - (defined(ERTS_SMP) && ERTS_POLL_USE_EPOLL) +#define ERTS_POLL_IS_FALLBACK (ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT) && ERTS_ENABLE_KERNEL_POLL -#define ERTS_POLL_COALESCE_KP_RES (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) +#define ERTS_POLL_USE_CONCURRENT_UPDATE (ERTS_POLL_USE_EPOLL || ERTS_POLL_USE_KQUEUE) -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -# define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 1 -#else -# define ERTS_POLL_ASYNC_INTERRUPT_SUPPORT 0 -#endif +#define ERTS_POLL_USE_WAKEUP_PIPE (!ERTS_POLL_USE_CONCURRENT_UPDATE) -#define ERTS_POLL_USE_WAKEUP_PIPE \ - (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS)) +#if !ERTS_POLL_USE_CONCURRENT_UPDATE -#ifdef ERTS_SMP +#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ + erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) +#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ + erts_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) +#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ + ((int) erts_atomic32_read_nob(&(PS)->have_update_requests)) #define ERTS_POLLSET_LOCK(PS) \ - erts_smp_mtx_lock(&(PS)->mtx) + erts_mtx_lock(&(PS)->mtx) #define ERTS_POLLSET_UNLOCK(PS) \ - erts_smp_mtx_unlock(&(PS)->mtx) - -#define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1)) -#define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0) -#define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_atomic32_read_nob(&(PS)->polled)) + erts_mtx_unlock(&(PS)->mtx) #else -#define ERTS_POLLSET_LOCK(PS) -#define ERTS_POLLSET_UNLOCK(PS) -#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0 -#define ERTS_POLLSET_UNSET_POLLED(PS) -#define ERTS_POLLSET_IS_POLLED(PS) 0 - -#endif - -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE -#define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1) -#define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0) -#define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ - ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests)) -#else #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0 -#endif -#if ERTS_POLL_USE_FALLBACK -# if ERTS_POLL_USE_POLL -# define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1) -# elif ERTS_POLL_USE_SELECT -# define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_select_fds > 1) -# endif +#define ERTS_POLLSET_LOCK(PS) +#define ERTS_POLLSET_UNLOCK(PS) + #endif + /* * --- Data types ------------------------------------------------------------ */ -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + #define ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE 128 typedef struct ErtsPollSetUpdateRequestsBlock_ ErtsPollSetUpdateRequestsBlock; @@ -222,206 +228,146 @@ struct ErtsPollSetUpdateRequestsBlock_ { int fds[ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE]; }; -#endif - - -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE # define ERTS_POLL_FD_FLG_INURQ (((unsigned short) 1) << 0) -#endif -#if ERTS_POLL_USE_FALLBACK -# define ERTS_POLL_FD_FLG_INFLBCK (((unsigned short) 1) << 1) -# define ERTS_POLL_FD_FLG_USEFLBCK (((unsigned short) 1) << 2) -#endif -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) -# define ERTS_POLL_FD_FLG_RST (((unsigned short) 1) << 3) -#endif +# define ERTS_POLL_FD_FLG_RST (((unsigned short) 1) << 1) + typedef struct { #if ERTS_POLL_USE_POLL int pix; #endif + ErtsPollEvents used_events; ErtsPollEvents events; -#if ERTS_POLL_COALESCE_KP_RES - unsigned short res_ev_ix; -#endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK unsigned short flags; -#endif } ErtsFdStatus; - -#if ERTS_POLL_COALESCE_KP_RES -/* res_ev_ix max value */ -#define ERTS_POLL_MAX_RES ((1 << sizeof(unsigned short)*8) - 1) -#endif - -#if ERTS_POLL_USE_KQUEUE - -#define ERTS_POLL_KQ_OP_HANDLED 1 -#define ERTS_POLL_KQ_OP_DEL_R 2 -#define ERTS_POLL_KQ_OP_DEL_W 3 -#define ERTS_POLL_KQ_OP_ADD_R 4 -#define ERTS_POLL_KQ_OP_ADD_W 5 -#define ERTS_POLL_KQ_OP_ADD2_R 6 -#define ERTS_POLL_KQ_OP_ADD2_W 7 - #endif -struct ErtsPollSet_ { - ErtsPollSet next; +/* + * This struct is not really exported, but it's nice to + * get unique names in debugger for kp/nkp + */ +struct ERTS_POLL_EXPORT(erts_pollset) { + int id; int internal_fd_limit; - ErtsFdStatus *fds_status; - erts_smp_atomic_t no_of_user_fds; - int fds_status_len; + erts_atomic_t no_of_user_fds; + #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; - int res_events_len; -#if ERTS_POLL_USE_EPOLL - struct epoll_event *res_events; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *res_events; -#elif ERTS_POLL_USE_DEVPOLL - struct pollfd *res_events; -#endif #endif /* ERTS_POLL_USE_KERNEL_POLL */ + #if ERTS_POLL_USE_POLL int next_poll_fds_ix; int no_poll_fds; int poll_fds_len; - struct pollfd*poll_fds; + struct pollfd *poll_fds; #elif ERTS_POLL_USE_SELECT int next_sel_fd; int max_fd; -#if ERTS_POLL_USE_FALLBACK - int no_select_fds; -#endif ERTS_fd_set input_fds; ERTS_fd_set res_input_fds; ERTS_fd_set output_fds; ERTS_fd_set res_output_fds; +#elif ERTS_POLL_USE_DEVPOLL + struct pollfd *poll_fds; + int poll_fds_ix; #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + ErtsFdStatus *fds_status; + int fds_status_len; ErtsPollSetUpdateRequestsBlock update_requests; ErtsPollSetUpdateRequestsBlock *curr_upd_req_block; - erts_smp_atomic32_t have_update_requests; -#endif -#ifdef ERTS_SMP - erts_atomic32_t polled; - erts_smp_mtx_t mtx; + erts_atomic32_t have_update_requests; + erts_mtx_t mtx; + erts_atomic32_t wakeup_state; #endif + #if ERTS_POLL_USE_WAKEUP_PIPE int wake_fds[2]; #endif -#if ERTS_POLL_USE_TIMERFD - int timer_fd; -#endif -#if ERTS_POLL_USE_FALLBACK - int fallback_used; -#endif -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - erts_atomic32_t wakeup_state; -#endif - erts_atomic64_t timeout_time; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_t no_avoided_wakeups; - erts_smp_atomic_t no_avoided_interrupts; - erts_smp_atomic_t no_interrupt_timed; -#endif }; void erts_silence_warn_unused_result(long unused); static void fatal_error(char *format, ...); -static void fatal_error_async_signal_safe(char *error_str); static int max_fds = -1; -static ErtsPollSet pollsets; -static erts_smp_spinlock_t pollsets_lock; #if ERTS_POLL_USE_POLL +#if !ERTS_POLL_IS_FALLBACK +static ERTS_INLINE short ev2pollev(ErtsPollEvents ev) +{ + return ERTS_POLL_EV_E2N(ev); +} + +static ERTS_INLINE ErtsPollEvents pollev2ev(short ev) +{ + return ERTS_POLL_EV_N2E(ev); +} + +#else /* ERTS_POLL_IS_FALLBACK */ + static ERTS_INLINE short ev2pollev(ErtsPollEvents ev) { -#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE - return ERTS_POLL_EV_E2N(ev); -#else /* Note, we only map events we are interested in */ short res_ev = (short) 0; if (ev & ERTS_POLL_EV_IN) - res_ev |= ERTS_POLL_EV_NKP_IN; + res_ev |= ERTS_POLL_EV_NKP_IN; if (ev & ERTS_POLL_EV_OUT) - res_ev |= ERTS_POLL_EV_NKP_OUT; + res_ev |= ERTS_POLL_EV_NKP_OUT; return res_ev; -#endif } static ERTS_INLINE ErtsPollEvents pollev2ev(short ev) { -#if !ERTS_POLL_USE_FALLBACK || ERTS_POLL_USE_KQUEUE - return ERTS_POLL_EV_N2E(ev); -#else /* Note, we only map events we are interested in */ ErtsPollEvents res_ev = (ErtsPollEvents) 0; if (ev & ERTS_POLL_EV_NKP_IN) - res_ev |= ERTS_POLL_EV_IN; + res_ev |= ERTS_POLL_EV_IN; if (ev & ERTS_POLL_EV_NKP_OUT) - res_ev |= ERTS_POLL_EV_OUT; + res_ev |= ERTS_POLL_EV_OUT; if (ev & ERTS_POLL_EV_NKP_ERR) - res_ev |= ERTS_POLL_EV_ERR; + res_ev |= ERTS_POLL_EV_ERR; if (ev & ERTS_POLL_EV_NKP_NVAL) - res_ev |= ERTS_POLL_EV_NVAL; - return res_ev; -#endif + res_ev |= ERTS_POLL_EV_NVAL; + return res_ev; } -#endif +#endif /* !ERTS_POLL_IS_FALLBACK */ + +#endif /* ERTS_POLL_USE_POLL */ + #ifdef HARD_DEBUG static void check_poll_result(ErtsPollResFd pr[], int len); -#if ERTS_POLL_USE_DEVPOLL -static void check_poll_status(ErtsPollSet ps); -#endif /* ERTS_POLL_USE_DEVPOLL */ #endif /* HARD_DEBUG */ -#ifdef ERTS_POLL_DEBUG_PRINT +#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG) +static void check_poll_status(ErtsPollSet *ps); +#endif /* ERTS_POLL_USE_DEVPOLL && DEBUG */ static void print_misc_debug_info(void); +#if ERTS_POLL_USE_EPOLL +uint32_t epoll_events(int kp_fd, int fd); #endif -static ERTS_INLINE void -init_timeout_time(ErtsPollSet ps) -{ - erts_atomic64_init_nob(&ps->timeout_time, - (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX); -} - -static ERTS_INLINE void -set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time) -{ - erts_atomic64_set_relb(&ps->timeout_time, - (erts_aint64_t) time); -} - -static ERTS_INLINE ErtsMonotonicTime -get_timeout_time(ErtsPollSet ps) -{ - return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time); -} #define ERTS_POLL_NOT_WOKEN 0 #define ERTS_POLL_WOKEN -1 #define ERTS_POLL_WOKEN_INTR 1 +#if !ERTS_POLL_USE_CONCURRENT_UPDATE static ERTS_INLINE void -reset_wakeup_state(ErtsPollSet ps) +reset_wakeup_state(ErtsPollSet *ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); -#endif } +#endif static ERTS_INLINE int -is_woken(ErtsPollSet ps) +is_woken(ErtsPollSet *ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +#if !ERTS_POLL_USE_CONCURRENT_UPDATE return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN; #else return 0; @@ -429,9 +375,9 @@ is_woken(ErtsPollSet ps) } static ERTS_INLINE int -is_interrupted_reset(ErtsPollSet ps) +is_interrupted_reset(ErtsPollSet *ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +#if !ERTS_POLL_USE_CONCURRENT_UPDATE return (erts_atomic32_xchg_acqb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) == ERTS_POLL_WOKEN_INTR); #else @@ -440,9 +386,9 @@ is_interrupted_reset(ErtsPollSet ps) } static ERTS_INLINE void -woke_up(ErtsPollSet ps) +woke_up(ErtsPollSet *ps) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +#if !ERTS_POLL_USE_CONCURRENT_UPDATE erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); if (wakeup_state == ERTS_POLL_NOT_WOKEN) (void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, @@ -459,28 +405,23 @@ woke_up(ErtsPollSet ps) #if ERTS_POLL_USE_WAKEUP_PIPE static ERTS_INLINE void -wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) +wake_poller(ErtsPollSet *ps, int interrupted) { +#if !ERTS_POLL_USE_CONCURRENT_UPDATE int wake; - if (async_signal_safe) - wake = 1; - else { - erts_aint32_t wakeup_state; - if (!interrupted) - wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, - ERTS_POLL_WOKEN, - ERTS_POLL_NOT_WOKEN); - else - wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, - ERTS_POLL_WOKEN_INTR); - wake = wakeup_state == ERTS_POLL_NOT_WOKEN; - } - /* - * NOTE: This function might be called from signal handlers in the - * non-smp case; therefore, it has to be async-signal safe in - * the non-smp case. - */ - if (wake) { + erts_aint32_t wakeup_state; + if (!interrupted) + wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + else + wakeup_state = erts_atomic32_xchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN_INTR); + wake = wakeup_state == ERTS_POLL_NOT_WOKEN; + + if (wake) +#endif + { ssize_t res; if (ps->wake_fds[1] < 0) return; /* Not initialized yet */ @@ -489,36 +430,27 @@ wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe) res = write(ps->wake_fds[1], "!", 1); } while (res < 0 && errno == EINTR); if (res <= 0 && errno != ERRNO_BLOCK) { - if (async_signal_safe) - fatal_error_async_signal_safe(__FILE__ - ":XXX:wake_poller(): " - "Failed to write on wakeup pipe\n"); - else - fatal_error("%s:%d:wake_poller(): " - "Failed to write to wakeup pipe fd=%d: " - "%s (%d)\n", - __FILE__, __LINE__, - ps->wake_fds[1], - erl_errno_id(errno), errno); + fatal_error("%s:%d:wake_poller(): " + "Failed to write to wakeup pipe fd=%d: " + "%s (%d)\n", + __FILE__, __LINE__, + ps->wake_fds[1], + erl_errno_id(errno), errno); } } } static ERTS_INLINE void -cleanup_wakeup_pipe(ErtsPollSet ps) +cleanup_wakeup_pipe(ErtsPollSet *ps) { -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT int intr = 0; -#endif int fd = ps->wake_fds[0]; int res; do { char buf[32]; res = read(fd, buf, sizeof(buf)); -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT if (res > 0) intr = 1; -#endif } while (res > 0 || (res < 0 && errno == EINTR)); if (res < 0 && errno != ERRNO_BLOCK) { fatal_error("%s:%d:cleanup_wakeup_pipe(): " @@ -528,14 +460,14 @@ cleanup_wakeup_pipe(ErtsPollSet ps) fd, erl_errno_id(errno), errno); } -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (intr) erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); #endif } static void -create_wakeup_pipe(ErtsPollSet ps) +create_wakeup_pipe(ErtsPollSet *ps) { int do_wake = 0; int wake_fds[2]; @@ -552,20 +484,13 @@ create_wakeup_pipe(ErtsPollSet ps) SET_NONBLOCKING(wake_fds[0]); SET_NONBLOCKING(wake_fds[1]); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("wakeup fds = {%d, %d}\n", wake_fds[0], wake_fds[1]); -#endif + DEBUG_PRINT("wakeup fds = {%d, %d}", ps, wake_fds[0], wake_fds[1]); ERTS_POLL_EXPORT(erts_poll_control)(ps, wake_fds[0], + ERTS_POLL_OP_ADD, ERTS_POLL_EV_IN, - 1, &do_wake); -#if ERTS_POLL_USE_FALLBACK - /* We depend on the wakeup pipe being handled by kernel poll */ - if (ps->fds_status[wake_fds[0]].flags & ERTS_POLL_FD_FLG_INFLBCK) - fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n", - __FILE__, __LINE__); -#endif + &do_wake); if (ps->internal_fd_limit <= wake_fds[1]) ps->internal_fd_limit = wake_fds[1] + 1; if (ps->internal_fd_limit <= wake_fds[0]) @@ -574,84 +499,16 @@ create_wakeup_pipe(ErtsPollSet ps) ps->wake_fds[1] = wake_fds[1]; } -#endif /* ERTS_POLL_USE_WAKEUP_PIPE */ - -/* - * --- timer fd ----------------------------------------------------------- - */ - -#if ERTS_POLL_USE_TIMERFD - -/* We use the timerfd when using epoll_wait to get high accuracy - timeouts, i.e. we want to sleep with < ms accuracy. */ - -static void -create_timerfd(ErtsPollSet ps) -{ - int do_wake = 0; - int timer_fd; - timer_fd = timerfd_create(CLOCK_MONOTONIC,0); - ERTS_POLL_EXPORT(erts_poll_control)(ps, - timer_fd, - ERTS_POLL_EV_IN, - 1, &do_wake); -#if ERTS_POLL_USE_FALLBACK - /* We depend on the wakeup pipe being handled by kernel poll */ - if (ps->fds_status[timer_fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - fatal_error("%s:%d:create_wakeup_pipe(): Internal error\n", - __FILE__, __LINE__); -#endif - if (ps->internal_fd_limit <= timer_fd) - ps->internal_fd_limit = timer_fd + 1; - ps->timer_fd = timer_fd; -} - -static ERTS_INLINE void -timerfd_set(ErtsPollSet ps, struct itimerspec *its) -{ -#ifdef DEBUG - struct itimerspec old_its; - int res; - res = timerfd_settime(ps->timer_fd, 0, its, &old_its); - ASSERT(res == 0); - ASSERT(old_its.it_interval.tv_sec == 0 && - old_its.it_interval.tv_nsec == 0 && - old_its.it_value.tv_sec == 0 && - old_its.it_value.tv_nsec == 0); - -#else - timerfd_settime(ps->timer_fd, 0, its, NULL); #endif -} - -static ERTS_INLINE int -timerfd_clear(ErtsPollSet ps, int res, int max_res) { - - struct itimerspec its; - /* we always have to clear the timer */ - its.it_interval.tv_sec = 0; - its.it_interval.tv_nsec = 0; - its.it_value.tv_sec = 0; - its.it_value.tv_nsec = 0; - timerfd_settime(ps->timer_fd, 0, &its, NULL); - - /* only timeout fd triggered */ - if (res == 1 && ps->res_events[0].data.fd == ps->timer_fd) - return 0; - - return res; -} - -#endif /* ERTS_POLL_USE_TIMERFD */ - /* * --- Poll set update requests ---------------------------------------------- */ -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE static ERTS_INLINE void -enqueue_update_request(ErtsPollSet ps, int fd) +enqueue_update_request(ErtsPollSet *ps, int fd) { ErtsPollSetUpdateRequestsBlock *urqbp; @@ -666,13 +523,11 @@ enqueue_update_request(ErtsPollSet ps, int fd) urqbp = ps->curr_upd_req_block; if (urqbp->len == ERTS_POLLSET_UPDATE_REQ_BLOCK_SIZE) { - ASSERT(!urqbp->next); urqbp = erts_alloc(ERTS_ALC_T_POLLSET_UPDREQ, sizeof(ErtsPollSetUpdateRequestsBlock)); - ps->curr_upd_req_block->next = urqbp; - ps->curr_upd_req_block = urqbp; - urqbp->next = NULL; + urqbp->next = ps->curr_upd_req_block; urqbp->len = 0; + ps->curr_upd_req_block = urqbp; } ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INURQ; @@ -680,29 +535,29 @@ enqueue_update_request(ErtsPollSet ps, int fd) } static ERTS_INLINE void -free_update_requests_block(ErtsPollSet ps, +free_update_requests_block(ErtsPollSet *ps, ErtsPollSetUpdateRequestsBlock *urqbp) { if (urqbp != &ps->update_requests) erts_free(ERTS_ALC_T_POLLSET_UPDREQ, (void *) urqbp); else { - urqbp->next = NULL; urqbp->len = 0; } } -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ /* * --- Growing poll set structures ------------------------------------------- */ -#ifndef ERTS_KERNEL_POLL_VERSION /* only one shared implementation */ +#if !ERTS_NO_KERNEL_POLL_VERSION || !ERTS_ENABLE_KERNEL_POLL +/* only one shared implementation */ #define ERTS_FD_TABLE_MIN_LENGTH 1024 #define ERTS_FD_TABLE_EXP_THRESHOLD (2048*1024) -int erts_poll_new_table_len (int old_len, int need_len) +int erts_poll_new_table_len(int old_len, int need_len) { int new_len; @@ -712,7 +567,7 @@ int erts_poll_new_table_len (int old_len, int need_len) } else { new_len = old_len; - do { + do { if (new_len < ERTS_FD_TABLE_EXP_THRESHOLD) new_len *= 2; else @@ -725,30 +580,9 @@ int erts_poll_new_table_len (int old_len, int need_len) } #endif -#if ERTS_POLL_USE_KERNEL_POLL -static void -grow_res_events(ErtsPollSet ps, int new_len) -{ - size_t new_size = sizeof( -#if ERTS_POLL_USE_EPOLL - struct epoll_event -#elif ERTS_POLL_USE_DEVPOLL - struct pollfd -#elif ERTS_POLL_USE_KQUEUE - struct kevent -#endif - ) * erts_poll_new_table_len(ps->res_events_len, new_len); - /* We do not need to save previously stored data */ - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, ps->res_events); - ps->res_events = erts_alloc(ERTS_ALC_T_POLL_RES_EVS, new_size); - ps->res_events_len = new_len; -} -#endif /* ERTS_POLL_USE_KERNEL_POLL */ - #if ERTS_POLL_USE_POLL static void -grow_poll_fds(ErtsPollSet ps, int min_ix) +grow_poll_fds(ErtsPollSet *ps, int min_ix) { int i; int new_len = erts_poll_new_table_len(ps->poll_fds_len, min_ix + 1); @@ -796,8 +630,9 @@ ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out) # define ensure_select_fds(fd, in, out) do {} while(0) #endif /* _DARWIN_UNLIMITED_SELECT */ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE static void -grow_fds_status(ErtsPollSet ps, int min_fd) +grow_fds_status(ErtsPollSet *ps, int min_fd) { int i; int new_len = erts_poll_new_table_len(ps->fds_status_len, min_fd + 1); @@ -816,430 +651,26 @@ grow_fds_status(ErtsPollSet ps, int min_fd) #endif ps->fds_status[i].used_events = (ErtsPollEvents) 0; ps->fds_status[i].events = (ErtsPollEvents) 0; -#if ERTS_POLL_COALESCE_KP_RES - ps->fds_status[i].res_ev_ix = (unsigned short) ERTS_POLL_MAX_RES; -#endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE || ERTS_POLL_USE_FALLBACK ps->fds_status[i].flags = (unsigned short) 0; -#endif } ps->fds_status_len = new_len; } +#endif /* * --- Selecting fd to poll on ----------------------------------------------- */ -#if ERTS_POLL_USE_FALLBACK -static int update_fallback_pollset(ErtsPollSet ps, int fd); -#endif - -static ERTS_INLINE int -need_update(ErtsPollSet ps, int fd) -{ -#if ERTS_POLL_USE_KERNEL_POLL - int reset; -#endif - - ASSERT(fd < ps->fds_status_len); - -#if ERTS_POLL_USE_KERNEL_POLL - reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST); - if (reset && !ps->fds_status[fd].used_events) { - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; - reset = 0; - } -#elif defined(ERTS_SMP) - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; -#endif - - if (ps->fds_status[fd].used_events != ps->fds_status[fd].events) - return 1; - -#if ERTS_POLL_USE_KERNEL_POLL - return reset; -#else - return 0; -#endif -} - -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - -#if ERTS_POLL_USE_KQUEUE -#define ERTS_POLL_MIN_BATCH_BUF_SIZE 128 -#else -#define ERTS_POLL_MIN_BATCH_BUF_SIZE 64 -#endif - -typedef struct { - int len; - int size; -#if ERTS_POLL_USE_DEVPOLL - struct pollfd *buf; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *buf; - struct kevent *ebuf; -#endif -} ErtsPollBatchBuf; - - -static ERTS_INLINE void -setup_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - bbp->len = 0; -#if ERTS_POLL_USE_DEVPOLL - bbp->size = ps->res_events_len; - bbp->buf = ps->res_events; -#elif ERTS_POLL_USE_KQUEUE - bbp->size = ps->res_events_len/2; - bbp->buf = ps->res_events; - bbp->ebuf = bbp->buf + bbp->size; -#endif -} - - -#if ERTS_POLL_USE_DEVPOLL - -static void -write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - ssize_t wres; - char *buf = (char *) bbp->buf; - size_t buf_size = sizeof(struct pollfd)*bbp->len; - - while (1) { - wres = write(ps->kp_fd, (void *) buf, buf_size); - if (wres < 0) { - if (errno == EINTR) - continue; - fatal_error("%s:%d:write_batch_buf(): " - "Failed to write to /dev/poll: " - "%s (%d)\n", - __FILE__, __LINE__, - erl_errno_id(errno), errno); - } - buf_size -= wres; - if (buf_size <= 0) - break; - buf += wres; - } - - if (buf_size < 0) { - fatal_error("%s:%d:write_devpoll_buf(): Internal error\n", - __FILE__, __LINE__); - } - bbp->len = 0; -} - -#elif ERTS_POLL_USE_KQUEUE - -static void -write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) -{ - int res; - int len = bbp->len; - struct kevent *buf = bbp->buf; - struct timespec ts = {0, 0}; - - do { - res = kevent(ps->kp_fd, buf, len, NULL, 0, &ts); - } while (res < 0 && errno == EINTR); - if (res < 0) { - int i; - struct kevent *ebuf = bbp->ebuf; - do { - res = kevent(ps->kp_fd, buf, len, ebuf, len, &ts); - } while (res < 0 && errno == EINTR); - if (res < 0) { - fatal_error("%s:%d: kevent() failed: %s (%d)\n", - __FILE__, __LINE__, erl_errno_id(errno), errno); - } - for (i = 0; i < res; i++) { - if (ebuf[i].flags & EV_ERROR) { - short filter; - int fd = (int) ebuf[i].ident; - - switch ((int) (long) ebuf[i].udata) { - - /* - * Since we use a lazy update approach EV_DELETE will - * frequently fail. This since kqueue automatically - * removes a file descriptor that is closed from the - * poll set. - */ - case ERTS_POLL_KQ_OP_DEL_R: - case ERTS_POLL_KQ_OP_DEL_W: - case ERTS_POLL_KQ_OP_HANDLED: - break; - - /* - * According to the kqueue man page EVFILT_READ support - * does not imply EVFILT_WRITE support; therefore, - * if an EV_ADD fail, we may have to remove other - * events on this fd in the kqueue pollset before - * adding fd to the fallback pollset. - */ - case ERTS_POLL_KQ_OP_ADD_W: - if (ps->fds_status[fd].used_events & ERTS_POLL_EV_IN) { - filter = EVFILT_READ; - goto rm_add_fb; - } - goto add_fb; - case ERTS_POLL_KQ_OP_ADD_R: - if (ps->fds_status[fd].used_events & ERTS_POLL_EV_OUT) { - filter = EVFILT_WRITE; - goto rm_add_fb; - } - goto add_fb; - case ERTS_POLL_KQ_OP_ADD2_W: - case ERTS_POLL_KQ_OP_ADD2_R: { - int j; - for (j = i+1; j < res; j++) { - if (fd == (int) ebuf[j].ident) { - ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED; - if (!(ebuf[j].flags & EV_ERROR)) { - switch ((int) (long) ebuf[j].udata) { - case ERTS_POLL_KQ_OP_ADD2_W: - filter = EVFILT_WRITE; - goto rm_add_fb; - case ERTS_POLL_KQ_OP_ADD2_R: - filter = EVFILT_READ; - goto rm_add_fb; - default: - fatal_error("%s:%d:write_batch_buf(): " - "Internal error", - __FILE__, __LINE__); - break; - } - } - goto add_fb; - } - } - /* The other add succeded... */ - filter = ((((int) (long) ebuf[i].udata) - == ERTS_POLL_KQ_OP_ADD2_W) - ? EVFILT_READ - : EVFILT_WRITE); - rm_add_fb: - { - struct kevent kev; - struct timespec ts = {0, 0}; - EV_SET(&kev, fd, filter, EV_DELETE, 0, 0, 0); - (void) kevent(ps->kp_fd, &kev, 1, NULL, 0, &ts); - } - - add_fb: - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - ASSERT(ps->fds_status[fd].used_events); - ps->fds_status[fd].used_events = 0; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - update_fallback_pollset(ps, fd); - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); - break; - } - default: - fatal_error("%s:%d:write_batch_buf(): Internal error", - __FILE__, __LINE__); - break; - } - } - } - } - bbp->len = 0; -} - -#endif /* ERTS_POLL_USE_KQUEUE */ - -static ERTS_INLINE void -batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) -{ - int buf_len; -#if ERTS_POLL_USE_DEVPOLL - short events; - struct pollfd *buf; -#elif ERTS_POLL_USE_KQUEUE - struct kevent *buf; -#endif - -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Doing lazy update on fd=%d\n", fd); -#endif - - if (!need_update(ps, fd)) - return; - - /* Make sure we have room for at least maximum no of entries - per fd */ - if (bbp->size - bbp->len < 2) - write_batch_buf(ps, bbp); - - buf_len = bbp->len; - buf = bbp->buf; - - ASSERT(fd < ps->fds_status_len); - -#if ERTS_POLL_USE_DEVPOLL - events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); - if (!events) { - buf[buf_len].events = POLLREMOVE; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - } - else if (!ps->fds_status[fd].used_events) { - buf[buf_len].events = events; - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); - } - else { - if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) - || (ps->fds_status[fd].used_events & ~events)) { - /* Reset or removed events... */ - buf[buf_len].fd = fd; - buf[buf_len].events = POLLREMOVE; - buf[buf_len++].revents = 0; - } - buf[buf_len].events = events; - } - buf[buf_len].fd = fd; - buf[buf_len++].revents = 0; - -#elif ERTS_POLL_USE_KQUEUE - - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) { - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK) - update_fallback_pollset(ps, fd); - else { /* Remove from fallback and try kqueue */ - ErtsPollEvents events = ps->fds_status[fd].events; - ps->fds_status[fd].events = (ErtsPollEvents) 0; - update_fallback_pollset(ps, fd); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - if (events) { - ps->fds_status[fd].events = events; - goto try_kqueue; - } - } - } - else { - ErtsPollEvents events, used_events; - int mod_w, mod_r; - try_kqueue: - events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); - used_events = ERTS_POLL_EV_E2N(ps->fds_status[fd].used_events); - if (!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) { - if (!used_events && - (events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) - goto do_add_rw; - mod_r = ((events & ERTS_POLL_EV_IN) - != (used_events & ERTS_POLL_EV_IN)); - mod_w = ((events & ERTS_POLL_EV_OUT) - != (used_events & ERTS_POLL_EV_OUT)); - goto do_mod; - } - else { /* Reset */ - if ((events & ERTS_POLL_EV_IN) && (events & ERTS_POLL_EV_OUT)) { - do_add_rw: - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_R); - buf_len++; - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD2_W); - buf_len++; - - } - else { - mod_r = 1; - mod_w = 1; - do_mod: - if (mod_r) { - if (events & ERTS_POLL_EV_IN) { - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD_R); - buf_len++; - } - else if (used_events & ERTS_POLL_EV_IN) { - EV_SET(&buf[buf_len], fd, EVFILT_READ, EV_DELETE, - 0, 0, (void *) ERTS_POLL_KQ_OP_DEL_R); - buf_len++; - } - } - if (mod_w) { - if (events & ERTS_POLL_EV_OUT) { - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_ADD, - 0, 0, (void *) ERTS_POLL_KQ_OP_ADD_W); - buf_len++; - } - else if (used_events & ERTS_POLL_EV_OUT) { - EV_SET(&buf[buf_len], fd, EVFILT_WRITE, EV_DELETE, - 0, 0, (void *) ERTS_POLL_KQ_OP_DEL_W); - buf_len++; - } - } - } - } - if (used_events) { - if (!events) { - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - } - } - else { - if (events) - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); - } - ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); - ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); - } - -#endif - - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; - ps->fds_status[fd].used_events = ps->fds_status[fd].events; - - bbp->len = buf_len; -} - -#else /* !ERTS_POLL_USE_BATCH_UPDATE_POLLSET */ - #if ERTS_POLL_USE_EPOLL static int -#if ERTS_POLL_USE_CONCURRENT_UPDATE -conc_update_pollset(ErtsPollSet ps, int fd, int *update_fallback) -#else -update_pollset(ErtsPollSet ps, int fd) -#endif +update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) { int res; - int op; + int epoll_op = EPOLL_CTL_MOD; struct epoll_event epe_templ; struct epoll_event epe; - ASSERT(fd < ps->fds_status_len); - - if (!need_update(ps, fd)) - return 0; - -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Doing update on fd=%d\n", fd); -#endif - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) { -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (!*update_fallback) { - *update_fallback = 1; - return 0; - } -#endif - if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK) { - return update_fallback_pollset(ps, fd); - } - else { /* Remove from fallback and try epoll */ - ErtsPollEvents events = ps->fds_status[fd].events; - ps->fds_status[fd].events = (ErtsPollEvents) 0; - res = update_fallback_pollset(ps, fd); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - if (!events) - return res; - ps->fds_status[fd].events = events; - } - } - - epe_templ.events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); + epe_templ.events = ERTS_POLL_EV_E2N(events) | EPOLLONESHOT; epe_templ.data.fd = fd; #ifdef VALGRIND @@ -1247,30 +678,24 @@ update_pollset(ErtsPollSet ps, int fd) memset((void *) &epe.data, 0, sizeof(epoll_data_t)); #endif - if (epe_templ.events && ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) { - do { - /* We init 'epe' every time since epoll_ctl() may modify it - (not declared const and not documented as const). */ - epe.events = epe_templ.events; - epe.data.fd = epe_templ.data.fd; - res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); - } while (res != 0 && errno == EINTR); - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - ps->fds_status[fd].used_events = 0; - } - - if (!epe_templ.events) { + switch (op) { + case ERTS_POLL_OP_DEL: /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ - op = EPOLL_CTL_DEL; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - } - else if (!ps->fds_status[fd].used_events) { - op = EPOLL_CTL_ADD; - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); - } - else { - op = EPOLL_CTL_MOD; + epoll_op = EPOLL_CTL_DEL; + epe_templ.events = 0; + erts_atomic_dec_nob(&ps->no_of_user_fds); + break; + case ERTS_POLL_OP_ADD: + epoll_op = EPOLL_CTL_ADD; + erts_atomic_inc_nob(&ps->no_of_user_fds); + break; + case ERTS_POLL_OP_MOD: + epoll_op = EPOLL_CTL_MOD; + break; + default: + ASSERT(0); + break; } do { @@ -1278,33 +703,32 @@ update_pollset(ErtsPollSet ps, int fd) (not declared const and not documented as const). */ epe.events = epe_templ.events; epe.data.fd = epe_templ.data.fd; - res = epoll_ctl(ps->kp_fd, op, fd, &epe); + res = epoll_ctl(ps->kp_fd, epoll_op, fd, &epe); } while (res != 0 && errno == EINTR); -#if defined(ERTS_POLL_DEBUG_PRINT) && 1 +#if ERTS_POLL_DEBUG_PRINT { int saved_errno = errno; - erts_printf("%s = epoll_ctl(%d, %s, %d, {Ox%x, %d})\n", - res == 0 ? "0" : erl_errno_id(errno), - ps->kp_fd, - (op == EPOLL_CTL_ADD - ? "EPOLL_CTL_ADD" - : (op == EPOLL_CTL_MOD - ? "EPOLL_CTL_MOD" - : (op == EPOLL_CTL_DEL - ? "EPOLL_CTL_DEL" - : "UNKNOWN"))), - fd, - epe_templ.events, - fd); + DEBUG_PRINT_FD("%s = epoll_ctl(%d, %s, %d, {0x%x, %d})", + ps, fd, + res == 0 ? "0" : erl_errno_id(errno), + ps->kp_fd, + (epoll_op == EPOLL_CTL_ADD + ? "EPOLL_CTL_ADD" + : (epoll_op == EPOLL_CTL_MOD + ? "EPOLL_CTL_MOD" + : (epoll_op == EPOLL_CTL_DEL + ? "EPOLL_CTL_DEL" + : "UNKNOWN"))), + fd, + epe_templ.events, + fd); errno = saved_errno; } #endif - if (res == 0) - ps->fds_status[fd].used_events = ps->fds_status[fd].events; - else { + if (res != 0) { switch (op) { - case EPOLL_CTL_MOD: + case ERTS_POLL_OP_MOD: epe.events = 0; do { /* We init 'epe' every time since epoll_ctl() may modify it @@ -1313,29 +737,18 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - ps->fds_status[fd].used_events = 0; /* Fall through ... */ - case EPOLL_CTL_ADD: { - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (!*update_fallback) { - *update_fallback = 1; - return 0; - } -#endif - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - res = update_fallback_pollset(ps, fd); - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); + case ERTS_POLL_OP_ADD: { + erts_atomic_dec_nob(&ps->no_of_user_fds); + res = ERTS_POLL_EV_NVAL; break; } - case EPOLL_CTL_DEL: { + case ERTS_POLL_OP_DEL: { /* * Since we use a lazy update approach EPOLL_CTL_DEL will * frequently fail. This since epoll automatically removes * a filedescriptor that is closed from the poll set. */ - ps->fds_status[fd].used_events = 0; res = 0; break; } @@ -1344,68 +757,278 @@ update_pollset(ErtsPollSet ps, int fd) __FILE__, __LINE__); break; } + } else { + res = events; } - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; return res; } -#if ERTS_POLL_USE_CONCURRENT_UPDATE +#endif /* ERTS_POLL_USE_EPOLL */ + +#if ERTS_POLL_USE_KQUEUE + +/* Some versions of the EV_SET macro used kevp multiple times, + so we define out own version that make sure that it is safe + to do kevp++ in the argument list. */ +#define ERTS_EV_SET(kevp, a, b, c, f) do { \ + struct kevent *kevp_ = kevp; \ + EV_SET(kevp_, a, b, c, 0, 0, f); \ + } while(0) + static int -update_pollset(ErtsPollSet ps, int fd) +update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) { - int update_fallback = 1; - return conc_update_pollset(ps, fd, &update_fallback); -} -#endif + int res = 0, len = 0; + struct kevent evts[2]; + struct timespec ts = {0, 0}; -#endif /* ERTS_POLL_USE_EPOLL */ +#ifdef EV_DISPATCH + /* If we have EV_DISPATCH we use it. The kevent descriptions for both + read and write are added on OP_ADD and removed on OP_DEL. And then + after than only EV_ENABLE|EV_DISPATCH are used. + + It could be possible to not modify the pollset when disabling and/or + deleting events, but that may cause the poll threads to be awoken + a lot more than they should so we take the cost here instead of + in the poll thread. + + Note: We need to have EV_DISPATCH both when the event is enabled and + disabled, as otherwise the event may be triggered twice on each re-arm. + Not sure if this is intended or not (can't find anything about it in the + man page), but it seems to be the way it works... + */ + + if (op == ERTS_POLL_OP_DEL) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + /* We could probably skip this delete, do we want to? */ + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE, (void *) 0); + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE, (void *) 0); + } else if (op == ERTS_POLL_OP_ADD) { + uint32_t flags; + erts_atomic_inc_nob(&ps->no_of_user_fds); + + flags = EV_ADD|EV_DISPATCH; + flags |= ((events & ERTS_POLL_EV_IN) ? 0 : EV_DISABLE); + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); + + flags = EV_ADD|EV_DISPATCH; + flags |= ((events & ERTS_POLL_EV_OUT) ? 0 : EV_DISABLE); + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } else { + uint32_t flags; + ASSERT(op == ERTS_POLL_OP_MOD); + + flags = EV_DISPATCH; + flags |= (events & ERTS_POLL_EV_IN) ? EV_ENABLE : EV_DISABLE; + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); + + flags = EV_DISPATCH; + flags |= (events & ERTS_POLL_EV_OUT) ? EV_ENABLE : EV_DISABLE; + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } +#else + uint32_t flags = EV_ADD|EV_ONESHOT; -#endif /* ERTS_POLL_USE_BATCH_UPDATE_POLLSET */ + if (op == ERTS_POLL_OP_DEL) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + /* We don't do anything when a delete is issued. The fds will be removed + when they are triggered, or when they are closed. */ + events = 0; + } else if (op == ERTS_POLL_OP_ADD) { + erts_atomic_inc_nob(&ps->no_of_user_fds); + } -#if ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK + if (events & ERTS_POLL_EV_IN) { + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, flags, (void *) ERTS_POLL_EV_IN); + } + if (events & ERTS_POLL_EV_OUT) { + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, flags, (void *) ERTS_POLL_EV_OUT); + } -#if ERTS_POLL_USE_FALLBACK -static int update_fallback_pollset(ErtsPollSet ps, int fd) -#else -static int update_pollset(ErtsPollSet ps, int fd) #endif -{ -#ifdef ERTS_POLL_DEBUG_PRINT -#if ERTS_POLL_USE_FALLBACK - erts_printf("Doing fallback update on fd=%d\n", fd); + if (len) + do { + res = kevent(ps->kp_fd, evts, len, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); +#if ERTS_POLL_DEBUG_PRINT + { + int saved_errno = errno, i; + char keventb[255], *keventbp = keventb; + if (res < 0) + keventbp += sprintf(keventbp,"%s = ",erl_errno_id(saved_errno)); + else + keventbp += sprintf(keventbp,"%d = ",res); + keventbp += sprintf(keventbp, "kevent(%d, {",ps->kp_fd); + for (i = 0; i < len; i++) { + const char *flags = "UNKNOWN"; + if (evts[i].flags == EV_DELETE) flags = "EV_DELETE"; + if (evts[i].flags == (EV_ADD|EV_ONESHOT)) flags = "EV_ADD|EV_ONESHOT"; +#ifdef EV_DISPATCH + if (evts[i].flags == (EV_ADD|EV_DISPATCH)) flags = "EV_ADD|EV_DISPATCH"; + if (evts[i].flags == (EV_ADD|EV_DISABLE)) flags = "EV_ADD|EV_DISABLE"; + if (evts[i].flags == (EV_ENABLE|EV_DISPATCH)) flags = "EV_ENABLE|EV_DISPATCH"; + if (evts[i].flags == EV_DISABLE) flags = "EV_DISABLE"; + if (evts[i].flags == (EV_DISABLE|EV_DISPATCH)) flags = "EV_DISABLE|EV_DISABLE"; +#endif + + keventbp += sprintf(keventbp, "%s{%lu, %s, %s}",i > 0 ? ", " : "", + evts[i].ident, + (evts[i].filter == EVFILT_READ + ? "EVFILT_READ" + : (evts[i].filter == EVFILT_WRITE + ? "EVFILT_WRITE" + : "UNKNOWN")), flags); + } + keventbp += sprintf(keventbp, "}, %d)", len); + DEBUG_PRINT_FD("%s", ps, fd, keventb); + errno = saved_errno; + } +#endif + if (res < 0) { + if (op != ERTS_POLL_OP_DEL) { +#ifdef EV_RECEIPT + struct kevent receipt_evts[2]; + len = 0; + ERTS_EV_SET(&evts[len++], fd, EVFILT_WRITE, EV_DELETE|EV_RECEIPT, (void *) 0); + ERTS_EV_SET(&evts[len++], fd, EVFILT_READ, EV_DELETE|EV_RECEIPT, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, len, receipt_evts, 2, &ts); + } while (res < 0 && errno == EINTR); #else - erts_printf("Doing update on fd=%d\n", fd); + ERTS_EV_SET(&evts[0], fd, EVFILT_WRITE, EV_DELETE, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); + ERTS_EV_SET(&evts[0], fd, EVFILT_READ, EV_DELETE, (void *) 0); + do { + res = kevent(ps->kp_fd, evts, 1, NULL, 0, &ts); + } while (res < 0 && errno == EINTR); +#endif + if (op == ERTS_POLL_OP_ADD) + erts_atomic_dec_nob(&ps->no_of_user_fds); + events = ERTS_POLL_EV_NVAL; + } else + events = 0; + } + return events; +} + +#endif /* ERTS_POLL_USE_KQUEUE */ + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + +static ERTS_INLINE void +init_batch_update(ErtsPollSet *ps, int len) +{ +#if ERTS_POLL_USE_DEVPOLL + ASSERT(ps->poll_fds == NULL); + ps->poll_fds = erts_alloc(ERTS_ALC_T_TMP, sizeof(struct pollfd) * len); + ps->poll_fds_ix = 0; #endif +} + +static ERTS_INLINE void +write_batch_update(ErtsPollSet *ps) +{ +#if ERTS_POLL_USE_DEVPOLL + ssize_t wres; + char *buf = (char *) ps->poll_fds; + size_t buf_size = sizeof(struct pollfd)*ps->poll_fds_ix; + + while (1) { + wres = write(ps->kp_fd, (void *) buf, buf_size); + if (wres < 0) { + if (errno == EINTR) + continue; + fatal_error("%s:%d:write_batch_buf(): " + "Failed to write to /dev/poll: " + "%s (%d)\n", + __FILE__, __LINE__, + erl_errno_id(errno), errno); + } +#if ERTS_POLL_DEBUG_PRINT + { + int saved_errno = errno, i; + char devpollb[2048], *devpollbp = devpollb; + devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd); + for (i = 0; i < wres / sizeof(struct pollfd); i++) { + if (devpollbp == devpollb) + devpollbp += sprintf(devpollbp, "%d = devpoll(%d, {", wres, ps->kp_fd); + devpollbp += sprintf(devpollbp, "%s{fd = %d, events = %s}", + i > 0 ? ", " : "", + ps->poll_fds[i].fd, + ev2str(ps->poll_fds[i].events)); + if (devpollbp - devpollb > 512) { + devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix); + DEBUG_PRINT("%s", ps, devpollb); + devpollbp = devpollb; + } + } + devpollbp += sprintf(devpollbp, "}, %d)", ps->poll_fds_ix); + DEBUG_PRINT("%s", ps, devpollb); + errno = saved_errno; + } #endif - ASSERT(fd < ps->fds_status_len); -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].used_events - ? (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - : (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_USEFLBCK)); + buf_size -= wres; + if (buf_size <= 0) + break; + buf += wres; + } + + if (buf_size < 0) { + fatal_error("%s:%d:write_devpoll_buf(): Internal error\n", + __FILE__, __LINE__); + } + erts_free(ERTS_ALC_T_TMP, ps->poll_fds); + ps->poll_fds = NULL; #endif +} - if (!need_update(ps, fd)) - return 0; +static ERTS_INLINE int +need_update(ErtsPollSet *ps, int fd, int *resetp) +{ + int reset; + ASSERT(fd < ps->fds_status_len); -#if ERTS_POLL_USE_FALLBACK + reset = (int) (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST); ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_RST; -#endif + + *resetp = reset; + + if (reset || ps->fds_status[fd].used_events != ps->fds_status[fd].events) + return 1; + + return 0; +} + +static int update_pollset(ErtsPollSet *ps, ErtsPollResFd pr[], int fd) +{ + int res = 0, reset = 0; + ErtsPollEvents events = ps->fds_status[fd].events; + ASSERT(fd < ps->fds_status_len); + + if (!need_update(ps, fd, &reset)) + return res; #if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ - if (!ps->fds_status[fd].events) { + if (!events) { int pix = ps->fds_status[fd].pix; int last_pix; + + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (poll)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } + if (pix < 0) { -#if ERTS_POLL_USE_FALLBACK - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); -#endif - return -1; + return res; } -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); -#endif - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); + erts_atomic_dec_nob(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1421,127 +1044,151 @@ static int update_pollset(ErtsPollSet ps, int fd) /* Clear this fd status */ ps->fds_status[fd].pix = -1; ps->fds_status[fd].used_events = (ErtsPollEvents) 0; -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK; -#endif + } else { int pix = ps->fds_status[fd].pix; if (pix < 0) { -#if ERTS_POLL_USE_FALLBACK - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) - || fd == ps->kp_fd); -#endif - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); + erts_atomic_inc_nob(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); ps->poll_fds[pix].fd = fd; ps->fds_status[fd].pix = pix; -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; -#endif } -#if ERTS_POLL_USE_FALLBACK - ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); -#endif - /* Events to be used in next poll */ - ps->poll_fds[pix].events = ev2pollev(ps->fds_status[fd].events); + ps->poll_fds[pix].events = ev2pollev(events); if (ps->poll_fds[pix].revents) { /* Remove result events that we should not poll for anymore */ ps->poll_fds[pix].revents &= ev2pollev(~(~ps->fds_status[fd].used_events - & ps->fds_status[fd].events)); + & events)); } /* Save events to be used in next poll */ - ps->fds_status[fd].used_events = ps->fds_status[fd].events; + ps->fds_status[fd].used_events = events; } - return 0; + return res; #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ - { - ErtsPollEvents events = ps->fds_status[fd].events; + if (!events) { + + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (select)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } + + ERTS_FD_CLR(fd, &ps->input_fds); + ERTS_FD_CLR(fd, &ps->output_fds); + + if (ps->fds_status[fd].used_events) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + ps->fds_status[fd].used_events = (ErtsPollEvents) 0; + } + + if (fd == ps->max_fd) { + int max = ps->max_fd; + for (max = ps->max_fd; max >= 0; max--) + if (ps->fds_status[max].used_events) + break; + ps->max_fd = max; + } + + } else { + ensure_select_fds(fd, &ps->input_fds, &ps->output_fds); - if ((ERTS_POLL_EV_IN & events) - != (ERTS_POLL_EV_IN & ps->fds_status[fd].used_events)) { - if (ERTS_POLL_EV_IN & events) { - ERTS_FD_SET(fd, &ps->input_fds); - } - else { - ERTS_FD_CLR(fd, &ps->input_fds); - } - } - if ((ERTS_POLL_EV_OUT & events) - != (ERTS_POLL_EV_OUT & ps->fds_status[fd].used_events)) { - if (ERTS_POLL_EV_OUT & events) { - ERTS_FD_SET(fd, &ps->output_fds); - } - else { - ERTS_FD_CLR(fd, &ps->output_fds); - } - } - if (!ps->fds_status[fd].used_events) { - ASSERT(events); - erts_smp_atomic_inc_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds++; - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; -#endif - } - else if (!events) { - ASSERT(ps->fds_status[fd].used_events); - erts_smp_atomic_dec_nob(&ps->no_of_user_fds); - ps->fds_status[fd].events = events; -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds--; - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INFLBCK; -#endif - } + if (!ps->fds_status[fd].used_events) + erts_atomic_inc_nob(&ps->no_of_user_fds); + + if (events & ERTS_POLL_EV_IN) + ERTS_FD_SET(fd, &ps->input_fds); + else + ERTS_FD_CLR(fd, &ps->input_fds); + + if (events & ERTS_POLL_EV_OUT) + ERTS_FD_SET(fd, &ps->output_fds); + else + ERTS_FD_CLR(fd, &ps->output_fds); ps->fds_status[fd].used_events = events; - if (events && fd > ps->max_fd) - ps->max_fd = fd; - else if (!events && fd == ps->max_fd) { - int max = ps->max_fd; - for (max = ps->max_fd; max >= 0; max--) - if (ps->fds_status[max].used_events) - break; - ps->max_fd = max; - } + if (fd > ps->max_fd) + ps->max_fd = fd; } - return 0; -#endif -} -#endif /* ERTS_POLL_USE_POLL || ERTS_POLL_USE_SELECT || ERTS_POLL_USE_FALLBACK */ + return res; +#elif ERTS_POLL_USE_DEVPOLL -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE + if (!events) { -static void -handle_update_requests(ErtsPollSet ps) -{ - ErtsPollSetUpdateRequestsBlock *urqbp = &ps->update_requests; -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - ErtsPollBatchBuf bb; - setup_batch_buf(ps, &bb); + if (reset) { + /* When a fd has been reset, we tell the caller of erts_poll_wait + this by setting the fd as ERTS_POLL_EV_NONE */ + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NONE); + DEBUG_PRINT_FD("trig %s (devpoll)", ps, fd, ev2str(ERTS_POLL_EV_NONE)); + res++; + } + + ps->poll_fds[ps->poll_fds_ix].fd = fd; + ps->poll_fds[ps->poll_fds_ix].revents = 0; + ps->poll_fds[ps->poll_fds_ix++].events = POLLREMOVE; + + if (ps->fds_status[fd].used_events) { + erts_atomic_dec_nob(&ps->no_of_user_fds); + ps->fds_status[fd].used_events = 0; + } + + } else { + if (!ps->fds_status[fd].used_events) { + erts_atomic_inc_nob(&ps->no_of_user_fds); + } + ps->poll_fds[ps->poll_fds_ix].fd = fd; + ps->poll_fds[ps->poll_fds_ix].revents = 0; + ps->poll_fds[ps->poll_fds_ix++].events = ERTS_POLL_EV_E2N(events); + ps->fds_status[fd].used_events = ps->fds_status[fd].events; + } + + return res; #endif +} + +static int +handle_update_requests(ErtsPollSet *ps, ErtsPollResFd pr[], int no_fds) +{ + int res = 0; + ErtsPollSetUpdateRequestsBlock *urqbp = ps->curr_upd_req_block; while (urqbp) { ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp; int i; int len = urqbp->len; + + init_batch_update(ps, len); + for (i = 0; i < len; i++) { int fd = urqbp->fds[i]; ASSERT(fd < ps->fds_status_len); - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ; -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - batch_update_pollset(ps, fd, &bb); -#else - update_pollset(ps, fd); -#endif + ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ); + + /* We have run out of PollResFd slots to put results in, + so we yield here and return later for more. */ + if (res == no_fds && pr != NULL) { + memmove(urqbp->fds, urqbp->fds+i, sizeof(int) * (len - i)); + urqbp->len -= i; + ps->curr_upd_req_block = urqbp; + write_batch_update(ps); + return res; + } + + if (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INURQ) { + ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_INURQ; + res += update_pollset(ps, pr + res, fd); + } } free_urqbp = urqbp; @@ -1549,12 +1196,9 @@ handle_update_requests(ErtsPollSet ps) free_update_requests_block(ps, free_urqbp); - } + write_batch_update(ps); -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - if (bb.len) - write_batch_buf(ps, &bb); -#endif + } ps->curr_upd_req_block = &ps->update_requests; @@ -1563,17 +1207,19 @@ handle_update_requests(ErtsPollSet ps) #endif ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(ps); + return res; } -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ static ERTS_INLINE ErtsPollEvents -poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake) +poll_control(ErtsPollSet *ps, int fd, ErtsPollOp op, + ErtsPollEvents events, int *do_wake) { ErtsPollEvents new_events; if (fd < ps->internal_fd_limit || fd >= max_fds) { - if (fd < 0) { + if (fd < 0 || fd >= max_fds) { new_events = ERTS_POLL_EV_ERR; goto done; } @@ -1589,124 +1235,49 @@ poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake goto done; } #endif -#if ERTS_POLL_USE_TIMERFD - if (fd == ps->timer_fd) { - new_events = ERTS_POLL_EV_NVAL; - goto done; - } -#endif } +#if ERTS_POLL_USE_CONCURRENT_UPDATE + + new_events = update_pollset(ps, fd, op, events); + +#else /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ if (fd >= ps->fds_status_len) grow_fds_status(ps, fd); ASSERT(fd < ps->fds_status_len); - new_events = ps->fds_status[fd].events; - - if (events == 0) { - *do_wake = 0; - goto done; - } - - if (on) - new_events |= events; - else - new_events &= ~events; - - if (new_events == (ErtsPollEvents) 0) { -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) - ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST; -#endif -#if ERTS_POLL_USE_FALLBACK - ps->fds_status[fd].flags &= ~ERTS_POLL_FD_FLG_USEFLBCK; -#endif + if (op == ERTS_POLL_OP_DEL) { + ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_RST; + ps->fds_status[fd].events = 0; + *do_wake = 1; + } else if (op == ERTS_POLL_OP_ADD) { + ASSERT(ps->fds_status[fd].events == 0); + ps->fds_status[fd].events = events; + *do_wake = 1; + } else { + ASSERT(op == ERTS_POLL_OP_MOD); + ps->fds_status[fd].events = events; + *do_wake = 1; } - - ps->fds_status[fd].events = new_events; - - if (new_events == ps->fds_status[fd].used_events -#if ERTS_POLL_USE_KERNEL_POLL || defined(ERTS_SMP) - && !(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) -#endif - ) { - *do_wake = 0; - goto done; - } - -#if !ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE - if (update_pollset(ps, fd) != 0) - new_events = ERTS_POLL_EV_ERR; -#else /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ - -#if ERTS_POLL_USE_CONCURRENT_UPDATE - if (ERTS_POLLSET_IS_POLLED(ps)) { - int update_fallback = 0; - conc_update_pollset(ps, fd, &update_fallback); - if (!update_fallback) { - *do_wake = 0; /* no need to wake kernel poller */ - goto done; - } - } -#endif + new_events = ps->fds_status[fd].events; enqueue_update_request(ps, fd); - -#ifdef ERTS_SMP - /* - * If new events have been added, we need to wake up the - * polling thread, but if events have been removed we don't. - */ - if ((new_events && (ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)) - || (~ps->fds_status[fd].used_events & new_events)) - *do_wake = 1; -#endif /* ERTS_SMP */ - -#endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ + +#endif /* !ERTS_POLL_USE_CONCURRENT_UPDATE */ done: -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("0x%x = poll_control(ps, %d, 0x%x, %s) do_wake=%d\n", - (int) new_events, fd, (int) events, (on ? "on" : "off"), *do_wake); -#endif + DEBUG_PRINT_FD("%s = %s(%p, %d, %s, %s) do_wake=%d", + ps, fd, ev2str(new_events), __FUNCTION__, ps, + fd, op2str(op), ev2str(events), *do_wake); return new_events; } -void -ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, - ErtsPollControlEntry pcev[], - int len) -{ - int i; - int do_wake; - int final_do_wake = 0; - - ERTS_POLLSET_LOCK(ps); - - for (i = 0; i < len; i++) { - do_wake = 0; - pcev[i].events = poll_control(ps, - pcev[i].fd, - pcev[i].events, - pcev[i].on, - &do_wake); - final_do_wake |= do_wake; - } - - ERTS_POLLSET_UNLOCK(ps); - -#ifdef ERTS_SMP - if (final_do_wake) - wake_poller(ps, 0, 0); -#endif /* ERTS_SMP */ - -} - ErtsPollEvents -ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps, ErtsSysFdType fd, + ErtsPollOp op, ErtsPollEvents events, - int on, int* do_wake) /* In: Wake up polling thread */ /* Out: Poller is woken */ { @@ -1714,15 +1285,15 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, ERTS_POLLSET_LOCK(ps); - res = poll_control(ps, fd, events, on, do_wake); + res = poll_control(ps, fd, op, events, do_wake); ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (*do_wake) { - wake_poller(ps, 0, 0); + wake_poller(ps, 0); } -#endif /* ERTS_SMP */ +#endif return res; } @@ -1734,188 +1305,64 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, #if ERTS_POLL_USE_KERNEL_POLL static ERTS_INLINE int -save_kp_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, int chk_fds_res) +ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf) { - int res = 0; - int i; - int n = chk_fds_res < max_res ? chk_fds_res : max_res; +#if !ERTS_POLL_USE_CONCURRENT_UPDATE || ERTS_POLL_DEBUG_PRINT || ERTS_POLL_USE_WAKEUP_PIPE + int n = chk_fds_res < max_res ? chk_fds_res : max_res, i; + int res = n; #if ERTS_POLL_USE_WAKEUP_PIPE int wake_fd = ps->wake_fds[0]; #endif -#if ERTS_POLL_USE_TIMERFD - int timer_fd = ps->timer_fd; -#endif for (i = 0; i < n; i++) { - -#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ - - if (ps->res_events[i].events) { - int fd = ps->res_events[i].data.fd; - int ix; - ErtsPollEvents revents; -#if ERTS_POLL_USE_WAKEUP_PIPE - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } + int fd = ERTS_POLL_RES_GET_FD(&pr[i]); +#ifdef DEBUG_PRINT_MODE + ErtsPollEvents evts = ERTS_POLL_RES_GET_EVTS(pr+i); #endif -#if ERTS_POLL_USE_TIMERFD - if (fd == timer_fd) { - continue; - } -#endif - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - /* epoll_wait() can repeat the same fd in result array... */ - ix = (int) ps->fds_status[fd].res_ev_ix; - ASSERT(ix >= 0); - if (ix >= res || pr[ix].fd != fd) { - ix = res; - pr[ix].fd = fd; - pr[ix].events = (ErtsPollEvents) 0; - } - revents = ERTS_POLL_EV_N2E(ps->res_events[i].events); - pr[ix].events |= revents; - if (revents) { - if (res == ix) { - ps->fds_status[fd].res_ev_ix = (unsigned short) ix; - res++; - } - } - } - -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ - - struct kevent *ev; - int fd; - int ix; - - ev = &ps->res_events[i]; - fd = (int) ev->ident; - ASSERT(fd < ps->fds_status_len); - ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)); - ix = (int) ps->fds_status[fd].res_ev_ix; - - ASSERT(ix >= 0); - if (ix >= res || pr[ix].fd != fd) { - ix = res; - pr[ix].fd = (int) ev->ident; - pr[ix].events = (ErtsPollEvents) 0; - } - - if (ev->filter == EVFILT_READ) { -#if ERTS_POLL_USE_WAKEUP_PIPE - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } + DEBUG_PRINT_FD("trig %s (%s)", ps, fd, + ev2str(evts), +#if ERTS_POLL_USE_KQUEUE + "kqueue" +#elif ERTS_POLL_USE_EPOLL + "epoll" +#else + "/dev/poll" #endif - pr[ix].events |= ERTS_POLL_EV_IN; - } - else if (ev->filter == EVFILT_WRITE) - pr[ix].events |= ERTS_POLL_EV_OUT; - if (ev->flags & (EV_ERROR|EV_EOF)) { - if ((ev->flags & EV_ERROR) && (((int) ev->data) == EBADF)) - pr[ix].events |= ERTS_POLL_EV_NVAL; - else - pr[ix].events |= ERTS_POLL_EV_ERR; - } - if (pr[ix].events) { - if (res == ix) { - ps->fds_status[fd].res_ev_ix = (unsigned short) ix; - res++; - } - } - -#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ + ); - if (ps->res_events[i].revents) { - int fd = ps->res_events[i].fd; - ErtsPollEvents revents; #if ERTS_POLL_USE_WAKEUP_PIPE - if (fd == wake_fd) { - cleanup_wakeup_pipe(ps); - continue; - } + if (fd == wake_fd) { + cleanup_wakeup_pipe(ps); + ERTS_POLL_RES_SET_EVTS(&pr[i], ERTS_POLL_EV_NONE); + if (n == 1) + return 0; + } #endif -#if ERTS_POLL_USE_TIMERFD - if (fd == timer_fd) { - continue; - } -#endif - revents = ERTS_POLL_EV_N2E(ps->res_events[i].events); - pr[res].fd = fd; - pr[res].events = revents; - res++; - } - +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + else { + /* Reset the events to emulate ONESHOT semantics */ + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); + } #endif - } return res; -} - -#endif /* ERTS_POLL_USE_KERNEL_POLL */ - -#if ERTS_POLL_USE_FALLBACK - -static int -get_kp_results(ErtsPollSet ps, ErtsPollResFd pr[], int max_res) -{ - int res; -#if ERTS_POLL_USE_KQUEUE - struct timespec ts = {0, 0}; -#endif - - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); - - do { -#if ERTS_POLL_USE_EPOLL - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0); -#elif ERTS_POLL_USE_KQUEUE - res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); -#endif - } while (res < 0 && errno == EINTR); - - if (res < 0) { - fatal_error("%s:%d: %s() failed: %s (%d)\n", - __FILE__, __LINE__, -#if ERTS_POLL_USE_EPOLL - "epoll_wait", -#elif ERTS_POLL_USE_KQUEUE - "kevent", +#else + ASSERT(chk_fds_res <= max_res); + return chk_fds_res; #endif - erl_errno_id(errno), errno); - } - - return save_kp_result(ps, pr, max_res, res); } -#endif /* ERTS_POLL_USE_FALLBACK */ - - +#else /* !ERTS_POLL_USE_KERNEL_POLL */ static ERTS_INLINE int -save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, - int chk_fds_res, int ebadf) +ERTS_POLL_EXPORT(save_result)(ErtsPollSet *ps, ErtsPollResFd pr[], int max_res, int chk_fds_res, int ebadf) { -#if ERTS_POLL_USE_DEVPOLL - return save_kp_result(ps, pr, max_res, chk_fds_res); -#elif ERTS_POLL_USE_FALLBACK - if (!ps->fallback_used) - return save_kp_result(ps, pr, max_res, chk_fds_res); - else -#endif /* ERTS_POLL_USE_FALLBACK */ - { - #if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ int res = 0; -#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; -#endif int i, first_ix, end_ix; /* @@ -1932,23 +1379,30 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (ps->poll_fds[i].revents != (short) 0) { int fd = ps->poll_fds[i].fd; ErtsPollEvents revents; -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, &pr[res], max_res-res); - i++; - continue; - } -#elif ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); i++; continue; } -#endif revents = pollev2ev(ps->poll_fds[i].revents); - pr[res].fd = fd; - pr[res].events = revents; + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], revents); + + /* If an fd returns as error, we may want to check the + update_requests queue to see if it has been reset + before delivering the result?!?! This should allow + the user to do driver_dselect + close without waiting + for stop_select... */ + + DEBUG_PRINT_FD("trig %s (poll)", ps, ERTS_POLL_RES_GET_FD(&pr[res]), + ev2str(ERTS_POLL_RES_GET_EVTS(&pr[res]))); + res++; + + /* Clear the events for this fd in order to mimic + how epoll ONESHOT works */ + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); } i++; } @@ -1964,9 +1418,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ int res = 0; -#if ERTS_POLL_USE_WAKEUP_PIPE && !ERTS_POLL_USE_FALLBACK int wake_fd = ps->wake_fds[0]; -#endif int fd, first_fd, end_fd; /* @@ -1979,29 +1431,23 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (!ebadf) { while (1) { while (fd < end_fd && res < max_res) { - - pr[res].events = (ErtsPollEvents) 0; + ErtsPollEvents events = 0; if (ERTS_FD_ISSET(fd, &ps->res_input_fds)) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, &pr[res], max_res-res); - fd++; - continue; - } -#elif ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].events |= ERTS_POLL_EV_IN; + events |= ERTS_POLL_EV_IN; } if (ERTS_FD_ISSET(fd, &ps->res_output_fds)) - pr[res].events |= ERTS_POLL_EV_OUT; - if (pr[res].events) { - pr[res].fd = fd; + events |= ERTS_POLL_EV_OUT; + if (events) { + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], events); res++; + ps->fds_status[fd].events = 0; + enqueue_update_request(ps, fd); } fd++; } @@ -2034,7 +1480,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, if (ps->fds_status[fd].events & ERTS_POLL_EV_OUT) { oset = &ps->res_output_fds; ERTS_FD_ZERO(oset); - ERTS_FD_SET(fd, oset); + ERTS_FD_SET(fd, oset); } do { /* Initiate 'tv' each time; @@ -2043,49 +1489,31 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, sres = ERTS_SELECT(ps->max_fd+1, iset, oset, NULL, &tv); } while (sres < 0 && errno == EINTR); if (sres < 0) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, - &pr[res], - max_res-res); - fd++; - continue; - } -#elif ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].fd = fd; - pr[res].events = ERTS_POLL_EV_NVAL; + ERTS_POLL_RES_SET_FD(&pr[res], fd); + ERTS_POLL_RES_SET_EVTS(&pr[res], ERTS_POLL_EV_NVAL); res++; } else if (sres > 0) { - pr[res].fd = fd; + ErtsPollEvents events = 0; + ERTS_POLL_RES_SET_FD(&pr[res], fd); if (iset && ERTS_FD_ISSET(fd, iset)) { -#if ERTS_POLL_USE_FALLBACK - if (fd == ps->kp_fd) { - res += get_kp_results(ps, - &pr[res], - max_res-res); - fd++; - continue; - } -#elif ERTS_POLL_USE_WAKEUP_PIPE if (fd == wake_fd) { cleanup_wakeup_pipe(ps); fd++; continue; } -#endif - pr[res].events |= ERTS_POLL_EV_IN; + events |= ERTS_POLL_EV_IN; } if (oset && ERTS_FD_ISSET(fd, oset)) { - pr[res].events |= ERTS_POLL_EV_OUT; + events |= ERTS_POLL_EV_OUT; } - ASSERT(pr[res].events); + ASSERT(events); + ERTS_POLL_RES_SET_EVTS(&pr[res], events); res++; } } @@ -2100,377 +1528,145 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, } ps->next_sel_fd = fd; return res; -#endif - } +#endif /* ERTS_POLL_USE_SELECT */ } -static ERTS_INLINE ErtsMonotonicTime -get_timeout(ErtsPollSet ps, - int resolution, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout, save_timeout_time; - - if (timeout_time == ERTS_POLL_NO_TIMEOUT) { - save_timeout_time = ERTS_MONOTONIC_TIME_MIN; - timeout = 0; - } - else { - ErtsMonotonicTime diff_time, current_time; - current_time = erts_get_monotonic_time(NULL); - diff_time = timeout_time - current_time; - if (diff_time <= 0) { - save_timeout_time = ERTS_MONOTONIC_TIME_MIN; - timeout = 0; - } - else { - save_timeout_time = current_time; - switch (resolution) { - case 1000: - /* Round up to nearest even milli second */ - timeout = ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1; - if (timeout > (ErtsMonotonicTime) INT_MAX) - timeout = (ErtsMonotonicTime) INT_MAX; - save_timeout_time += ERTS_MSEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000); - break; - case 1000000: - /* Round up to nearest even micro second */ - timeout = ERTS_MONOTONIC_TO_USEC(diff_time - 1) + 1; - save_timeout_time += ERTS_USEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000); - break; - case 1000000000: - /* Round up to nearest even nano second */ - timeout = ERTS_MONOTONIC_TO_NSEC(diff_time - 1) + 1; - save_timeout_time += ERTS_NSEC_TO_MONOTONIC(timeout); - timeout -= ERTS_PREMATURE_TIMEOUT(timeout, 1000*1000*1000); - break; - default: - ERTS_INTERNAL_ERROR("Invalid resolution"); - timeout = 0; - save_timeout_time = 0; - break; - } - } - } - set_timeout_time(ps, save_timeout_time); - return timeout; -} - -#if ERTS_POLL_USE_SELECT +#endif /* !ERTS_POLL_USE_KERNEL_POLL */ static ERTS_INLINE int -get_timeout_timeval(ErtsPollSet ps, - SysTimeval *tvp, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout = get_timeout(ps, - 1000*1000, - timeout_time); - - if (!timeout) { - tvp->tv_sec = 0; - tvp->tv_usec = 0; - - return 0; - } - else { - ErtsMonotonicTime sec = timeout/(1000*1000); - tvp->tv_sec = sec; - tvp->tv_usec = timeout - sec*(1000*1000); - - ASSERT(tvp->tv_sec >= 0); - ASSERT(tvp->tv_usec >= 0); - ASSERT(tvp->tv_usec < 1000*1000); - - return !0; - } - -} - -#endif - -#if ERTS_POLL_USE_KQUEUE || (ERTS_POLL_USE_POLL && defined(HAVE_PPOLL)) || ERTS_POLL_USE_TIMERFD - -static ERTS_INLINE int -get_timeout_timespec(ErtsPollSet ps, - struct timespec *tsp, - ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime timeout = get_timeout(ps, - 1000*1000*1000, - timeout_time); - - if (!timeout) { - tsp->tv_sec = 0; - tsp->tv_nsec = 0; - return 0; - } - else { - ErtsMonotonicTime sec = timeout/(1000*1000*1000); - tsp->tv_sec = sec; - tsp->tv_nsec = timeout - sec*(1000*1000*1000); - - ASSERT(tsp->tv_sec >= 0); - ASSERT(tsp->tv_nsec >= 0); - ASSERT(tsp->tv_nsec < 1000*1000*1000); - - return !0; - } -} - -#endif - -#if ERTS_POLL_USE_TIMERFD - -static ERTS_INLINE int -get_timeout_itimerspec(ErtsPollSet ps, - struct itimerspec *itsp, - ErtsMonotonicTime timeout_time) -{ - - itsp->it_interval.tv_sec = 0; - itsp->it_interval.tv_nsec = 0; - - return get_timeout_timespec(ps, &itsp->it_value, timeout_time); -} - -#endif - -static ERTS_INLINE int -check_fd_events(ErtsPollSet ps, ErtsMonotonicTime timeout_time, int max_res) +check_fd_events(ErtsPollSet *ps, ErtsPollResFd pr[], int do_wait, int max_res) { int res; - ERTS_MSACC_PUSH_STATE_M(); - if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0 - && timeout_time == ERTS_POLL_NO_TIMEOUT) { - /* Nothing to poll and zero timeout; done... */ - return 0; - } - else { - int timeout; -#if ERTS_POLL_USE_FALLBACK - if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) { - -#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); -#if ERTS_POLL_USE_TIMERFD - { - struct itimerspec its; - timeout = get_timeout_itimerspec(ps, &its, timeout_time); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - timerfd_set(ps, &its); - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, -1); - res = timerfd_clear(ps, res, max_res); - } else { - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, 0); - } - } -#else /* !ERTS_POLL_USE_TIMERFD */ - timeout = (int) get_timeout(ps, 1000, timeout_time); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = epoll_wait(ps->kp_fd, ps->res_events, max_res, timeout); -#endif /* !ERTS_POLL_USE_TIMERFD */ -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ - struct timespec ts; - if (max_res > ps->res_events_len) - grow_res_events(ps, max_res); - timeout = get_timeout_timespec(ps, &ts, timeout_time); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts); -#endif /* ----------------------------------------- */ - } - else /* use fallback (i.e. poll() or select()) */ -#endif /* ERTS_POLL_USE_FALLBACK */ - { + int timeout = do_wait ? -1 : 0; + DEBUG_PRINT_WAIT("Entering check_fd_events(), do_wait=%d", ps, do_wait); + { +#if ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ + res = epoll_wait(ps->kp_fd, pr, max_res, timeout); + +#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ + struct timespec ts = {0, 0}; + struct timespec *tsp = timeout ? NULL : &ts; + res = kevent(ps->kp_fd, NULL, 0, pr, max_res, tsp); +#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ + /* + * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds + * is set too high. dp_nfds should not be set greater than + * the maximum number of file descriptors in the poll set. + */ + struct dvpoll poll_res; + int nfds = (int) erts_atomic_read_nob(&ps->no_of_user_fds) + 1 /* wakeup pipe */; + poll_res.dp_nfds = nfds < max_res ? nfds : max_res; + poll_res.dp_fds = pr; + poll_res.dp_timeout = timeout; + res = ioctl(ps->kp_fd, DP_POLL, &poll_res); -#if ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ - /* - * The ioctl() will fail with EINVAL on Solaris 10 if dp_nfds - * is set too high. dp_nfds should not be set greater than - * the maximum number of file descriptors in the poll set. - */ - struct dvpoll poll_res; - int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_WAKEUP_PIPE - nfds++; /* Wakeup pipe */ -#endif - timeout = (int) get_timeout(ps, 1000, timeout_time); - poll_res.dp_nfds = nfds < max_res ? nfds : max_res; - if (poll_res.dp_nfds > ps->res_events_len) - grow_res_events(ps, poll_res.dp_nfds); - poll_res.dp_fds = ps->res_events; - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - poll_res.dp_timeout = timeout; - res = ioctl(ps->kp_fd, DP_POLL, &poll_res); -#elif ERTS_POLL_USE_POLL && defined(HAVE_PPOLL) /* --- ppoll ---------------- */ - struct timespec ts; - timeout = get_timeout_timespec(ps, &ts, timeout_time); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = ppoll(ps->poll_fds, ps->no_poll_fds, &ts, NULL); #elif ERTS_POLL_USE_POLL /* --- poll --------------------------------- */ - timeout = (int) get_timeout(ps, 1000, timeout_time); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = poll(ps->poll_fds, ps->no_poll_fds, timeout); + res = poll(ps->poll_fds, ps->no_poll_fds, timeout); + #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ - SysTimeval to; - timeout = get_timeout_timeval(ps, &to, timeout_time); + SysTimeval tv = {0, 0}; + SysTimeval *tvp = timeout ? NULL : &tv; - ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); - ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); + ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); + ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); - } - res = ERTS_SELECT(ps->max_fd + 1, - &ps->res_input_fds, - &ps->res_output_fds, - NULL, - &to); -#ifdef ERTS_SMP - if (timeout) { - erts_thr_progress_finalize_wait(NULL); - ERTS_MSACC_POP_STATE_M(); - } - if (res < 0 - && errno == EBADF - && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { - /* - * This may have happened because another thread deselected - * a fd in our poll set and then closed it, i.e. the driver - * behaved correctly. We wan't to avoid looking for a bad - * fd, that may even not exist anymore. Therefore, handle - * update requests and try again. - * - * We don't know how much of the timeout is left; therfore, - * we use a zero timeout. If no error occur and no events - * have triggered, we fake an EAGAIN error and let the caller - * restart us. - */ - to.tv_sec = 0; - to.tv_usec = 0; - ERTS_POLLSET_LOCK(ps); - handle_update_requests(ps); - ERTS_POLLSET_UNLOCK(ps); - res = ERTS_SELECT(ps->max_fd + 1, - &ps->res_input_fds, - &ps->res_output_fds, - NULL, - &to); - if (res == 0) { - errno = EAGAIN; - res = -1; - } - } -#endif /* ERTS_SMP */ - return res; + res = ERTS_SELECT(ps->max_fd + 1, + &ps->res_input_fds, + &ps->res_output_fds, + NULL, + tvp); #endif /* ----------------------------------------- */ - } - if (timeout) { -#ifdef ERTS_SMP - erts_thr_progress_finalize_wait(NULL); -#endif - ERTS_MSACC_POP_STATE_M(); - } - return res; } + DEBUG_PRINT_WAIT("Leaving check_fd_events(), res=%d", ps, res); + return res; } int -ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps, ErtsPollResFd pr[], - int *len, - ErtsMonotonicTime timeout_time) + int *len) { - ErtsMonotonicTime to; - int res, no_fds; + int res, no_fds, used_fds = 0; int ebadf = 0; -#ifdef ERTS_SMP + int do_wait; int ps_locked = 0; -#endif + ERTS_MSACC_DECLARE_CACHE(); no_fds = *len; -#ifdef ERTS_POLL_MAX_RES - if (no_fds >= ERTS_POLL_MAX_RES) - no_fds = ERTS_POLL_MAX_RES; -#endif - *len = 0; + ASSERT(no_fds > 0); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Entering erts_poll_wait(), timeout_time=%bps\n", - timeout_time); -#endif - - if (ERTS_POLLSET_SET_POLLED_CHK(ps)) { - res = EINVAL; /* Another thread is in erts_poll_wait() - on this pollset... */ - goto done; - } - - to = (is_woken(ps) - ? ERTS_POLL_NO_TIMEOUT /* Use zero timeout */ - : timeout_time); - -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { ERTS_POLLSET_LOCK(ps); - handle_update_requests(ps); + used_fds = handle_update_requests(ps, pr, no_fds); ERTS_POLLSET_UNLOCK(ps); + + if (used_fds == no_fds) { + *len = used_fds; + return 0; + } } #endif + do_wait = !is_woken(ps) && used_fds == 0; + + DEBUG_PRINT_WAIT("Entering %s(), do_wait=%d", ps, __FUNCTION__, do_wait); + + if (do_wait) { + erts_thr_progress_prepare_wait(NULL); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); + } + while (1) { - res = check_fd_events(ps, to, no_fds); + res = check_fd_events(ps, pr + used_fds, do_wait, no_fds - used_fds); + +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + if (res < 0 + && errno == EBADF + && ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) { + /* + * This may have happened because another thread deselected + * a fd in our poll set and then closed it, i.e. the driver + * behaved correctly. We wan't to avoid looking for a bad + * fd, that may even not exist anymore. Therefore, handle + * update requests and try again. This behaviour should only + * happen when using SELECT as the polling mechanism. + */ + ERTS_POLLSET_LOCK(ps); + used_fds += handle_update_requests(ps, pr + used_fds, no_fds - used_fds); + if (used_fds == no_fds) { + *len = used_fds; + ERTS_POLLSET_UNLOCK(ps); + return 0; + } + res = check_fd_events(ps, pr + used_fds, 0, no_fds - used_fds); + /* Keep the lock over the non-blocking poll in order to not + get any nasty races happening. */ + ERTS_POLLSET_UNLOCK(ps); + if (res == 0) { + errno = EAGAIN; + res = -1; + } + } +#endif + if (res != 0) break; - if (to == ERTS_POLL_NO_TIMEOUT) - break; - if (erts_get_monotonic_time(NULL) >= timeout_time) - break; + if (!do_wait) + break; + } + + if (do_wait) { + erts_thr_progress_finalize_wait(NULL); + ERTS_MSACC_UPDATE_CACHE(); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO); } woke_up(ps); - if (res == 0) { - res = ETIMEDOUT; - } - else if (res < 0) { + if (res < 0) { #if ERTS_POLL_USE_SELECT if (errno == EBADF) { ebadf = 1; @@ -2484,33 +1680,24 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, save_results: #endif -#ifdef ERTS_SMP ps_locked = 1; ERTS_POLLSET_LOCK(ps); -#endif - no_fds = save_poll_result(ps, pr, no_fds, res, ebadf); + used_fds += ERTS_POLL_EXPORT(save_result)(ps, pr + used_fds, no_fds - used_fds, res, ebadf); #ifdef HARD_DEBUG - check_poll_result(pr, no_fds); + check_poll_result(pr, used_fds); #endif - res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0); - *len = no_fds; + res = (used_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0); + *len = used_fds; } -#ifdef ERTS_SMP if (ps_locked) ERTS_POLLSET_UNLOCK(ps); - ERTS_POLLSET_UNSET_POLLED(ps); -#endif - done: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX); -#ifdef ERTS_POLL_DEBUG_PRINT - erts_printf("Leaving %s = erts_poll_wait()\n", - res == 0 ? "0" : erl_errno_id(res)); -#endif + DEBUG_PRINT_WAIT("Leaving %s = %s(len = %d)", ps, + res == 0 ? "0" : erl_errno_id(res), __FUNCTION__, *len); return res; } @@ -2520,54 +1707,13 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, */ void -ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) +ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set) { -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT +#if !ERTS_POLL_USE_CONCURRENT_UPDATE if (!set) reset_wakeup_state(ps); else - wake_poller(ps, 1, 0); -#endif -} - -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT -void -ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps) -{ - /* - * NOTE: This function is called from signal handlers, it, - * therefore, it has to be async-signal safe. - */ - wake_poller(ps, 1, 1); -} -#endif - -/* - * erts_poll_interrupt_timed(): - * If 'set' != 0, interrupt thread blocked in erts_poll_wait() if it - * is not guaranteed that it will timeout before 'msec' milli seconds. - */ -void -ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, - int set, - ErtsMonotonicTime timeout_time) -{ -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) - if (!set) - reset_wakeup_state(ps); - else { - ErtsMonotonicTime max_wait_time = get_timeout_time(ps); - if (max_wait_time > timeout_time) - wake_poller(ps, 1, 0); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - else { - if (ERTS_POLLSET_IS_POLLED(ps)) - erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups); - erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts); - } - erts_smp_atomic_inc_nob(&ps->no_interrupt_timed); -#endif - } + wake_poller(ps, 1); #endif } @@ -2581,13 +1727,19 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void) */ void -ERTS_POLL_EXPORT(erts_poll_init)(void) +ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_updates) { - erts_smp_spinlock_init(&pollsets_lock, "pollsets_lock"); - pollsets = NULL; errno = 0; + if (concurrent_updates) { +#if ERTS_POLL_USE_CONCURRENT_UPDATE + *concurrent_updates = 1; +#else + *concurrent_updates = 0; +#endif + } + #if !defined(NO_SYSCONF) max_fds = sysconf(_SC_OPEN_MAX); #elif ERTS_POLL_USE_SELECT @@ -2606,37 +1758,28 @@ ERTS_POLL_EXPORT(erts_poll_init)(void) fatal_error("erts_poll_init(): Failed to get max number of files: %s\n", erl_errno_id(errno)); -#ifdef ERTS_POLL_DEBUG_PRINT print_misc_debug_info(); -#endif } -ErtsPollSet -ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) +ErtsPollSet * +ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id) { #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; #endif - ErtsPollSet ps = erts_alloc(ERTS_ALC_T_POLLSET, - sizeof(struct ErtsPollSet_)); + ErtsPollSet *ps = erts_alloc(ERTS_ALC_T_POLLSET, + sizeof(struct ERTS_POLL_EXPORT(erts_pollset))); + ps->id = id; ps->internal_fd_limit = 0; - ps->fds_status = NULL; - ps->fds_status_len = 0; - erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0); + erts_atomic_init_nob(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL kp_fd = epoll_create(256); - ps->res_events_len = 0; - ps->res_events = NULL; #elif ERTS_POLL_USE_DEVPOLL kp_fd = open("/dev/poll", O_RDWR); - ps->res_events_len = 0; - ps->res_events = NULL; #elif ERTS_POLL_USE_KQUEUE kp_fd = kqueue(); - ps->res_events_len = 0; - ps->res_events = NULL; #endif if (kp_fd < 0) fatal_error("erts_poll_create_pollset(): Failed to " @@ -2650,10 +1793,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ": %s (%d)\n", erl_errno_id(errno), errno); #endif /* ERTS_POLL_USE_KERNEL_POLL */ -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - /* res_events is also used as write buffer */ - grow_res_events(ps, ERTS_POLL_MIN_BATCH_BUF_SIZE); -#endif #if ERTS_POLL_USE_POLL ps->next_poll_fds_ix = 0; ps->no_poll_fds = 0; @@ -2662,9 +1801,6 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #elif ERTS_POLL_USE_SELECT ps->next_sel_fd = 0; ps->max_fd = -1; -#if ERTS_POLL_USE_FALLBACK - ps->no_select_fds = 0; -#endif #ifdef _DARWIN_UNLIMITED_SELECT ps->input_fds.sz = 0; ps->input_fds.ptr = NULL; @@ -2681,133 +1817,66 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ERTS_FD_ZERO(&ps->res_output_fds); #endif #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + ps->fds_status = NULL; + ps->fds_status_len = 0; ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; - erts_smp_atomic32_init_nob(&ps->have_update_requests, 0); -#endif -#ifdef ERTS_SMP - erts_atomic32_init_nob(&ps->polled, 0); - erts_smp_mtx_init(&ps->mtx, "pollset"); -#endif -#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); -#endif -#if ERTS_POLL_USE_WAKEUP_PIPE - create_wakeup_pipe(ps); -#endif -#if ERTS_POLL_USE_TIMERFD - create_timerfd(ps); -#endif -#if ERTS_POLL_USE_FALLBACK - if (kp_fd >= ps->fds_status_len) - grow_fds_status(ps, kp_fd); - /* Force kernel poll fd into fallback (poll/select) set */ - ps->fds_status[kp_fd].flags - |= ERTS_POLL_FD_FLG_INFLBCK|ERTS_POLL_FD_FLG_USEFLBCK; - { - int do_wake = 0; - ERTS_POLL_EXPORT(erts_poll_control)(ps, kp_fd, ERTS_POLL_EV_IN, 1, - &do_wake); - } + erts_atomic32_init_nob(&ps->have_update_requests, 0); + erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); #endif #if ERTS_POLL_USE_KERNEL_POLL if (ps->internal_fd_limit <= kp_fd) ps->internal_fd_limit = kp_fd + 1; ps->kp_fd = kp_fd; #endif - init_timeout_time(ps); -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0); - erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0); - erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0); -#endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE - handle_update_requests(ps); -#endif -#if ERTS_POLL_USE_FALLBACK - ps->fallback_used = 0; -#endif - erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ - - erts_smp_spin_lock(&pollsets_lock); - ps->next = pollsets; - pollsets = ps; - erts_smp_spin_unlock(&pollsets_lock); - - return ps; -} - -void -ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) -{ - - if (ps->fds_status) - erts_free(ERTS_ALC_T_FD_STATUS, (void *) ps->fds_status); - -#if ERTS_POLL_USE_EPOLL - if (ps->kp_fd >= 0) - close(ps->kp_fd); - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events); -#elif ERTS_POLL_USE_DEVPOLL - if (ps->kp_fd >= 0) - close(ps->kp_fd); - if (ps->res_events) - erts_free(ERTS_ALC_T_POLL_RES_EVS, (void *) ps->res_events); -#elif ERTS_POLL_USE_POLL - if (ps->poll_fds) - erts_free(ERTS_ALC_T_POLL_FDS, (void *) ps->poll_fds); -#elif ERTS_POLL_USE_SELECT -#ifdef _DARWIN_UNLIMITED_SELECT - if (ps->input_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->input_fds.ptr); - if (ps->res_input_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_input_fds.ptr); - if (ps->output_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->output_fds.ptr); - if (ps->res_output_fds.ptr) - erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr); -#endif +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0); + create_wakeup_pipe(ps); + handle_update_requests(ps, NULL, 0); + cleanup_wakeup_pipe(ps); #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if ERTS_POLL_USE_KERNEL_POLL && (defined(__DARWIN__) || defined(__APPLE__) && defined(__MACH__)) { - ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; - while (urqbp) { - ErtsPollSetUpdateRequestsBlock *free_urqbp = urqbp; - urqbp = urqbp->next; - free_update_requests_block(ps, free_urqbp); - } - } -#endif -#ifdef ERTS_SMP - erts_smp_mtx_destroy(&ps->mtx); -#endif -#if ERTS_POLL_USE_WAKEUP_PIPE - if (ps->wake_fds[0] >= 0) - close(ps->wake_fds[0]); - if (ps->wake_fds[1] >= 0) - close(ps->wake_fds[1]); -#endif -#if ERTS_POLL_USE_TIMERFD - if (ps->timer_fd >= 0) - close(ps->timer_fd); -#endif + /* + * Using kqueue on OS X is a mess of brokenness... + * + * On OS X version older than 15.6 (i.e. OS X El Capitan released in July 2015), + * a thread waiting in kevent is not woken if an event is inserted into the kqueue + * by another thread and the event becomes ready. However if a new call to kevent + * is done by the waiting thread, the new event is found. + * + * So on effected OS X versions we could trigger the wakeup pipe so that + * the waiters will be woken and re-issue the kevent. However... + * + * On OS X version older then 16 (i.e. OS X Sierra released in September 2016), + * running the emulator driver_SUITE smp_select testcase consistently causes a + * kernel panic. I don't know why or what events that trigger it. But it seems + * like updates of the pollset while another thread is sleeping in it Creates + * some kind of race that triggers the kernel panic. + * + * So to deal with this, the erts configure check what OS X version is run + * and only enabled kernel poll on OS X 16 or newer. In addition, if someone + * attempts to compile Erlang on OS X 16 and then run it on OS X 15, we do the + * run-time check below to disallow this. + */ + int major, minor, build; + os_version(&major,&minor,&build); + if (major < 16) { + erts_fprintf(stderr,"BROKEN KQUEUE!\n" + "Erlang has been compiled with kernel-poll support,\n" + "but this OS X version is known to have kernel bugs\n" + "when using kernel-poll. You have two options:\n" + " 1) update to a newer OS X version (OS X Sierra or newer)\n" + " 2) recompile erlang without kernel-poll support\n"); + erts_exit(1, ""); + } + } +#endif + erts_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ - erts_smp_spin_lock(&pollsets_lock); - if (ps == pollsets) - pollsets = pollsets->next; - else { - ErtsPollSet prev_ps; - for (prev_ps = pollsets; ps != prev_ps->next; prev_ps = prev_ps->next) - ; - ASSERT(ps == prev_ps->next); - prev_ps->next = ps->next; - } - erts_smp_spin_unlock(&pollsets_lock); - - erts_free(ERTS_ALC_T_POLLSET, (void *) ps); + return ps; } /* @@ -2815,24 +1884,18 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) */ void -ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) +ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, ErtsPollInfo *pip) { -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE int pending_updates; #endif Uint size = 0; ERTS_POLLSET_LOCK(ps); - size += sizeof(struct ErtsPollSet_); + size += sizeof(struct ERTS_POLL_EXPORT(erts_pollset)); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE size += ps->fds_status_len*sizeof(ErtsFdStatus); - -#if ERTS_POLL_USE_EPOLL - size += ps->res_events_len*sizeof(struct epoll_event); -#elif ERTS_POLL_USE_DEVPOLL - size += ps->res_events_len*sizeof(struct pollfd); -#elif ERTS_POLL_USE_KQUEUE - size += ps->res_events_len*sizeof(struct kevent); #endif #if ERTS_POLL_USE_POLL @@ -2844,7 +1907,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif #endif -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE { ErtsPollSetUpdateRequestsBlock *urqbp = ps->update_requests.next; pending_updates = ps->update_requests.len; @@ -2856,7 +1919,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) } #endif - pip->primary = + pip->primary = #if ERTS_POLL_USE_KQUEUE "kqueue" #elif ERTS_POLL_USE_EPOLL @@ -2870,17 +1933,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif ; - pip->fallback = -#if !ERTS_POLL_USE_FALLBACK - NULL -#elif ERTS_POLL_USE_POLL - "poll" -#elif ERTS_POLL_USE_SELECT - "select" -#endif - ; - - pip->kernel_poll = + pip->kernel_poll = #if !ERTS_POLL_USE_KERNEL_POLL NULL #elif ERTS_POLL_USE_KQUEUE @@ -2894,34 +1947,13 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds); -#if ERTS_POLL_USE_WAKEUP_PIPE + pip->poll_set_size = (int) erts_atomic_read_nob(&ps->no_of_user_fds); +#if !ERTS_POLL_USE_CONCURRENT_UPDATE pip->poll_set_size++; /* Wakeup pipe */ #endif -#if ERTS_POLL_USE_TIMERFD - pip->poll_set_size++; /* timerfd */ -#endif - - pip->fallback_poll_set_size = -#if !ERTS_POLL_USE_FALLBACK - 0 -#elif ERTS_POLL_USE_POLL - ps->no_poll_fds -#elif ERTS_POLL_USE_SELECT - ps->no_select_fds -#endif - ; - -#if ERTS_POLL_USE_FALLBACK - /* If only kp_fd is in fallback poll set we don't use fallback... */ - if (pip->fallback_poll_set_size == 1) - pip->fallback_poll_set_size = 0; - else - pip->poll_set_size++; /* kp_fd */ -#endif pip->lazy_updates = -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE 1 #else 0 @@ -2929,21 +1961,13 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) ; pip->pending_updates = -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE +#if !ERTS_POLL_USE_CONCURRENT_UPDATE pending_updates #else 0 #endif ; - pip->batch_updates = -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET - 1 -#else - 0 -#endif - ; - pip->concurrent_updates = #if ERTS_POLL_USE_CONCURRENT_UPDATE 1 @@ -2952,13 +1976,23 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #endif ; - pip->max_fds = max_fds; + pip->is_fallback = +#if ERTS_POLL_IS_FALLBACK + 1 +#else + 0 +#endif + ; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups); - pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts); - pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed); + pip->batch_updates = +#if ERTS_POLL_USE_DEVPOLL + 1 +#else + 0 #endif + ; + + pip->max_fds = max_fds; ERTS_POLLSET_UNLOCK(ps); @@ -2994,35 +2028,60 @@ fatal_error(char *format, ...) abort(); } -static void -fatal_error_async_signal_safe(char *error_str) +/* + * --- Debug ----------------------------------------------------------------- + */ + +#if ERTS_POLL_USE_EPOLL +uint32_t epoll_events(int kp_fd, int fd) { - if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) { - /* See comment above in fatal_error() */ - return; + /* For epoll we read the information about what is selected upon from the proc fs.*/ + char fname[30]; + FILE *f; + unsigned int pos, flags, mnt_id; + int line = 0; + sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), kp_fd); + f = fopen(fname,"r"); + if (!f) { + fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno); + ASSERT(0); + return 0; } - if (error_str) { - int len = 0; - while (error_str[len]) - len++; - if (len) { - /* async signal safe */ - erts_silence_warn_unused_result(write(2, error_str, len)); - } + if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) { + fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno); + ASSERT(0); + return 0; } - abort(); + if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id)); + line += 3; + while (!feof(f)) { + /* tfd: 10 events: 40000019 data: 180000000a */ + int ev_fd; + uint32_t events; + uint64_t data; + if (fscanf(f,"tfd:%d events:%x data:%lx\n", &ev_fd, &events, &data) != 3) { + fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", fname, + line, + errno); + return 0; + } + if (fd == ev_fd) { + fclose(f); + return events; + } + } + fclose(f); + return 0; } - -/* - * --- Debug ----------------------------------------------------------------- - */ +#endif void -ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, +ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, ErtsPollEvents ev[], int len) { int fd; +#if !ERTS_POLL_USE_CONCURRENT_UPDATE ERTS_POLLSET_LOCK(ps); for (fd = 0; fd < len; fd++) { if (fd >= ps->fds_status_len) @@ -3030,12 +2089,7 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, else { ev[fd] = ps->fds_status[fd].events; if ( -#if ERTS_POLL_USE_WAKEUP_PIPE fd == ps->wake_fds[0] || fd == ps->wake_fds[1] || -#endif -#if ERTS_POLL_USE_TIMERFD - fd == ps->timer_fd || -#endif #if ERTS_POLL_USE_KERNEL_POLL fd == ps->kp_fd || #endif @@ -3044,7 +2098,50 @@ ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet ps, } } ERTS_POLLSET_UNLOCK(ps); +#elif ERTS_POLL_USE_EPOLL + /* For epoll we read the information about what is selected upon from the proc fs.*/ + char fname[30]; + FILE *f; + unsigned int pos, flags, mnt_id; + int line = 0; + sprintf(fname,"/proc/%d/fdinfo/%d",getpid(), ps->kp_fd); + for (fd = 0; fd < len; fd++) + ev[fd] = ERTS_POLL_EV_NONE; + f = fopen(fname,"r"); + if (!f) { + fprintf(stderr,"failed to open file %s, errno = %d\n", fname, errno); + return; + } + if (fscanf(f,"pos:\t%x\nflags:\t%x", &pos, &flags) != 2) { + fprintf(stderr,"failed to parse file %s, errno = %d\n", fname, errno); + ASSERT(0); + return; + } + if (fscanf(f,"\nmnt_id:\t%x\n", &mnt_id)); + line += 3; + while (!feof(f)) { + /* tfd: 10 events: 40000019 data: 180000000a */ + int fd; + uint32_t events; + uint64_t data; + if (fscanf(f,"tfd:%d events:%x data:%lx\n", &fd, &events, &data) != 3) { + fprintf(stderr,"failed to parse file %s on line %d, errno = %d\n", + fname, line, errno); + ASSERT(0); + return; + } + data &= 0xFFFFFFFF; + ASSERT(fd == data); + /* Events are the events that are being monitored, which of course include + error and hup events, but we are only interested in IN/OUT events */ + ev[fd] = (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) & ERTS_POLL_EV_N2E(events); + line++; + } +#else + for (fd = 0; fd < len; fd++) + ev[fd] = ERTS_POLL_EV_NONE; +#endif } #ifdef HARD_DEBUG @@ -3064,10 +2161,10 @@ check_poll_result(ErtsPollResFd pr[], int len) } -#if ERTS_POLL_USE_DEVPOLL +#if ERTS_POLL_USE_DEVPOLL && defined(DEBUG) static void -check_poll_status(ErtsPollSet ps) +check_poll_status(ErtsPollSet *ps) { int i; for (i = 0; i < ps->fds_status_len; i++) { @@ -3099,34 +2196,24 @@ check_poll_status(ErtsPollSet ps) #endif /* ERTS_POLL_USE_DEVPOLL */ #endif /* HARD_DEBUG */ -#ifdef ERTS_POLL_DEBUG_PRINT static void print_misc_debug_info(void) { - erts_printf("erts_poll using: %s lazy_updates:%s batch_updates:%s\n", +#if ERTS_POLL_DEBUG_PRINT + erts_printf("erts_poll using: %s lazy_updates:%s\n", #if ERTS_POLL_USE_KQUEUE "kqueue" #elif ERTS_POLL_USE_EPOLL "epoll" #elif ERTS_POLL_USE_DEVPOLL "/dev/poll" -#endif -#if ERTS_POLL_USE_FALLBACK - "-" -#endif -#if ERTS_POLL_USE_POLL +#elif ERTS_POLL_USE_POLL "poll" #elif ERTS_POLL_USE_SELECT "select" #endif , -#if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE - "true" -#else - "false" -#endif - , -#if ERTS_POLL_USE_BATCH_UPDATE_POLLSET +#if !ERTS_POLL_USE_CONCURRENT_UPDATE "true" #else "false" @@ -3145,6 +2232,20 @@ print_misc_debug_info(void) #ifdef FD_SETSIZE erts_printf("FD_SETSIZE=%d\n", FD_SETSIZE); #endif +#endif +} + +#ifdef ERTS_ENABLE_LOCK_COUNT +void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *pollset, int enable) +{ +#if !ERTS_POLL_USE_CONCURRENT_UPDATE + if(enable) { + erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL, + ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO); + } else { + erts_lcnt_uninstall(&pollset->mtx.lcnt); + } +#endif + return; } - #endif diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h index c16122610d..e9a667cac1 100644 --- a/erts/emulator/sys/common/erl_poll.h +++ b/erts/emulator/sys/common/erl_poll.h @@ -18,11 +18,31 @@ * %CopyrightEnd% */ -/* - * Description: Poll interface suitable for ERTS with or without - * SMP support. +/** + * @description: Poll interface suitable for ERTS with SMP support. + * + * @author: Rickard Green + * @author: Lukas Larsson + * + * This header file exports macros and functions that are used to + * react to I/O polling events from file descriptors or wait-able + * objects. The API exported is the following: * - * Author: Rickard Green + * defines: + * ERTS_POLL_EV_NONE - No events have been set. This is not the same as 0. + * ERTS_POLL_EV_IN - Represent an IN event + * ERTS_POLL_EV_OUT - Represent an OUT event + * ERTS_POLL_EV_ERR - Represent an error event + * ERTS_POLL_EV_NVAL - Represent an invalid event + * + * macro functions: + * ErtsSysFdType ERTS_POLL_RES_GET_FD(ErtsPollResFd *evt); + * void ERTS_POLL_RES_SET_FD(ErtsPollResFd *evt, ErtsSysFdType fd); + * ErtsPollEvents ERTS_POLL_RES_GET_EVTS(ErtsPollResFd *evt) + * void ERTS_POLL_RES_SET_EVTS(ErtsPollResFd *evt, ErtsPollEvents fd); + * + * functions: + * See erl_poll_api.h */ #ifndef ERL_POLL_H__ @@ -32,33 +52,27 @@ #define ERTS_POLL_NO_TIMEOUT ERTS_MONOTONIC_TIME_MIN -#if 0 -#define ERTS_POLL_COUNT_AVOIDED_WAKEUPS -#endif - #ifdef ERTS_ENABLE_KERNEL_POLL -# if defined(ERTS_KERNEL_POLL_VERSION) -# define ERTS_POLL_EXPORT(FUNC) FUNC ## _kp +# undef ERTS_ENABLE_KERNEL_POLL +# define ERTS_ENABLE_KERNEL_POLL 1 +# if defined(ERTS_NO_KERNEL_POLL_VERSION) +# define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk +# undef ERTS_NO_KERNEL_POLL_VERSION +# define ERTS_NO_KERNEL_POLL_VERSION 1 +# define ERTS_KERNEL_POLL_VERSION 0 # else -# define ERTS_POLL_EXPORT(FUNC) FUNC ## _nkp -# undef ERTS_POLL_DISABLE_KERNEL_POLL -# define ERTS_POLL_DISABLE_KERNEL_POLL +# undef ERTS_KERNEL_POLL_VERSION +# define ERTS_KERNEL_POLL_VERSION 1 +# define ERTS_NO_KERNEL_POLL_VERSION 0 +# define ERTS_POLL_EXPORT(FUNC) FUNC # endif #else # define ERTS_POLL_EXPORT(FUNC) FUNC -# undef ERTS_POLL_DISABLE_KERNEL_POLL -# define ERTS_POLL_DISABLE_KERNEL_POLL -#endif - -#ifdef ERTS_POLL_DISABLE_KERNEL_POLL -# undef HAVE_SYS_EPOLL_H -# undef HAVE_SYS_EVENT_H -# undef HAVE_SYS_DEVPOLL_H +# define ERTS_ENABLE_KERNEL_POLL 0 +# define ERTS_NO_KERNEL_POLL_VERSION 1 +# define ERTS_KERNEL_POLL_VERSION 0 #endif -#undef ERTS_POLL_USE_KERNEL_POLL -#define ERTS_POLL_USE_KERNEL_POLL 0 - #undef ERTS_POLL_USE_KQUEUE #define ERTS_POLL_USE_KQUEUE 0 #undef ERTS_POLL_USE_EPOLL @@ -70,68 +84,96 @@ #undef ERTS_POLL_USE_SELECT #define ERTS_POLL_USE_SELECT 0 -#if defined(HAVE_SYS_EVENT_H) -# undef ERTS_POLL_USE_KQUEUE -# define ERTS_POLL_USE_KQUEUE 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 -#elif defined(HAVE_SYS_EPOLL_H) -# undef ERTS_POLL_USE_EPOLL -# define ERTS_POLL_USE_EPOLL 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 -#elif defined(HAVE_SYS_DEVPOLL_H) -# undef ERTS_POLL_USE_DEVPOLL -# define ERTS_POLL_USE_DEVPOLL 1 -# undef ERTS_POLL_USE_KERNEL_POLL -# define ERTS_POLL_USE_KERNEL_POLL 1 +/* Defines which structure that erts_poll_wait should use to wait with + and how events should be represented */ +#define ERTS_POLL_USE_EPOLL_EVS 0 +#define ERTS_POLL_USE_KQUEUE_EVS 0 +#define ERTS_POLL_USE_DEVPOLL_EVS 0 +#define ERTS_POLL_USE_POLL_EVS 0 +#define ERTS_POLL_USE_SELECT_EVS 0 + +#define ERTS_POLL_USE_KERNEL_POLL ERTS_KERNEL_POLL_VERSION + +#if ERTS_ENABLE_KERNEL_POLL +# if defined(HAVE_SYS_EVENT_H) +# undef ERTS_POLL_USE_KQUEUE_EVS +# define ERTS_POLL_USE_KQUEUE_EVS 1 +# undef ERTS_POLL_USE_KQUEUE +# define ERTS_POLL_USE_KQUEUE ERTS_KERNEL_POLL_VERSION +# elif defined(HAVE_SYS_EPOLL_H) +# undef ERTS_POLL_USE_EPOLL_EVS +# define ERTS_POLL_USE_EPOLL_EVS 1 +# undef ERTS_POLL_USE_EPOLL +# define ERTS_POLL_USE_EPOLL ERTS_KERNEL_POLL_VERSION +# elif defined(HAVE_SYS_DEVPOLL_H) +# undef ERTS_POLL_USE_DEVPOLL_EVS +# define ERTS_POLL_USE_DEVPOLL_EVS 1 +# undef ERTS_POLL_USE_DEVPOLL +# define ERTS_POLL_USE_DEVPOLL ERTS_KERNEL_POLL_VERSION +# else +# error "Missing kernel poll implementation of erts_poll()" +# endif #endif -#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) - -#if !ERTS_POLL_USE_KERNEL_POLL || ERTS_POLL_USE_FALLBACK +#if ERTS_NO_KERNEL_POLL_VERSION # if defined(ERTS_USE_POLL) +# undef ERTS_POLL_USE_POLL_EVS +# define ERTS_POLL_USE_POLL_EVS 1 # undef ERTS_POLL_USE_POLL # define ERTS_POLL_USE_POLL 1 # elif !defined(__WIN32__) +# undef ERTS_POLL_USE_SELECT_EVS +# define ERTS_POLL_USE_SELECT_EVS 1 # undef ERTS_POLL_USE_SELECT # define ERTS_POLL_USE_SELECT 1 # endif #endif -#define ERTS_POLL_USE_TIMERFD 0 +#define ERTS_POLL_USE_FALLBACK (ERTS_POLL_USE_KQUEUE || ERTS_POLL_USE_EPOLL) typedef Uint32 ErtsPollEvents; -#undef ERTS_POLL_EV_E2N + +typedef enum { + ERTS_POLL_OP_ADD = 0, /* Add the FD to the pollset */ + ERTS_POLL_OP_MOD = 1, /* Modify the FD in the pollset */ + ERTS_POLL_OP_DEL = 2 /* Delete the FD from the pollset */ +} ErtsPollOp; + +#define op2str(op) (op == ERTS_POLL_OP_ADD ? "add" : \ + (op == ERTS_POLL_OP_MOD ? "mod" : "del")) #if defined(__WIN32__) /* --- win32 --------------------------------------- */ -#define ERTS_POLL_EV_IN 1 -#define ERTS_POLL_EV_OUT 2 -#define ERTS_POLL_EV_ERR 4 -#define ERTS_POLL_EV_NVAL 8 +#define ERTS_POLL_EV_IN 1 +#define ERTS_POLL_EV_OUT 2 +#define ERTS_POLL_EV_ERR 4 +#define ERTS_POLL_EV_NVAL 8 -#elif ERTS_POLL_USE_EPOLL /* --- epoll ------------------------------- */ +#define ERTS_POLL_EV_E2N(EV) (EV) +#define ERTS_POLL_EV_N2E(EV) (EV) -#include <sys/epoll.h> +#elif ERTS_POLL_USE_EPOLL_EVS /* --- epoll ------------------------------- */ -#ifdef HAVE_SYS_TIMERFD_H -#include <sys/timerfd.h> -#undef ERTS_POLL_USE_TIMERFD -#define ERTS_POLL_USE_TIMERFD 1 -#endif +#include <sys/epoll.h> #define ERTS_POLL_EV_E2N(EV) \ ((uint32_t) (EV)) #define ERTS_POLL_EV_N2E(EV) \ - ((ErtsPollEvents) (EV)) + ((ErtsPollEvents) (EV) & ~EPOLLONESHOT) #define ERTS_POLL_EV_IN ERTS_POLL_EV_N2E(EPOLLIN) #define ERTS_POLL_EV_OUT ERTS_POLL_EV_N2E(EPOLLOUT) #define ERTS_POLL_EV_NVAL ERTS_POLL_EV_N2E(EPOLLET) #define ERTS_POLL_EV_ERR ERTS_POLL_EV_N2E(EPOLLERR|EPOLLHUP) -#elif ERTS_POLL_USE_DEVPOLL /* --- devpoll ----------------------------- */ +typedef struct epoll_event ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->data.fd)) +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->data.fd = ident +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts) + +#elif ERTS_POLL_USE_DEVPOLL_EVS /* --- devpoll ----------------------------- */ #include <sys/devpoll.h> @@ -145,12 +187,37 @@ typedef Uint32 ErtsPollEvents; #define ERTS_POLL_EV_NVAL ERTS_POLL_EV_N2E(POLLNVAL) #define ERTS_POLL_EV_ERR ERTS_POLL_EV_N2E(POLLERR|POLLHUP) -#elif ERTS_POLL_USE_KQUEUE /* --- kqueue ------------------------------ */ +typedef struct pollfd ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->fd)) +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = ident +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->revents) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->revents = ERTS_POLL_EV_E2N(evts) + +#elif ERTS_POLL_USE_KQUEUE_EVS /* --- kqueue ------------------------------ */ /* Kqueue use fallback defines (poll() or select()) */ + +#include <sys/event.h> + +#ifdef ERTS_USE_POLL +# undef ERTS_POLL_USE_POLL_EVS +# define ERTS_POLL_USE_POLL_EVS 1 +#elif !defined(__WIN32__) +# undef ERTS_POLL_USE_SELECT_EVS +# define ERTS_POLL_USE_SELECT_EVS 1 #endif -#if ERTS_POLL_USE_POLL /* --- poll -------------------------------- */ +typedef struct kevent ErtsPollResFd; + +#define ERTS_POLL_RES_GET_FD(evt) ((ErtsSysFdType)((evt)->ident)) +#define ERTS_POLL_RES_SET_FD(evt, fd) (evt)->ident = fd +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((ErtsPollEvents)(evt)->udata) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->udata = (void*)(UWord)(ERTS_POLL_EV_E2N(evts)) +#endif + +#if ERTS_POLL_USE_POLL_EVS + /* --- poll -------------------------------- */ #include <poll.h> #define ERTS_POLL_EV_NKP_E2N(EV) \ @@ -169,7 +236,7 @@ typedef Uint32 ErtsPollEvents; #define ERTS_POLL_EV_NKP_NVAL ERTS_POLL_EV_N2E(POLLNVAL) #define ERTS_POLL_EV_NKP_ERR ERTS_POLL_EV_N2E(POLLERR|POLLHUP) -#elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ +#elif ERTS_POLL_USE_SELECT_EVS /* --- select ------------------------------ */ #define ERTS_POLL_EV_NKP_E2N(EV) (EV) #define ERTS_POLL_EV_NKP_N2E(EV) (EV) @@ -195,69 +262,65 @@ typedef Uint32 ErtsPollEvents; #endif -typedef struct ErtsPollSet_ *ErtsPollSet; - -typedef struct { - ErtsSysFdType fd; - ErtsPollEvents events; - int on; -} ErtsPollControlEntry; +#if !ERTS_ENABLE_KERNEL_POLL -typedef struct { +typedef struct _ErtsPollResFd { ErtsSysFdType fd; ErtsPollEvents events; } ErtsPollResFd; +#define ERTS_POLL_RES_GET_FD(evt) (evt)->fd +#define ERTS_POLL_RES_SET_FD(evt, ident) (evt)->fd = (ident) +#define ERTS_POLL_RES_GET_EVTS(evt) ERTS_POLL_EV_N2E((evt)->events) +#define ERTS_POLL_RES_SET_EVTS(evt, evts) (evt)->events = ERTS_POLL_EV_E2N(evts) + +#endif + +#define ERTS_POLL_EV_NONE (UINT_MAX & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT|ERTS_POLL_EV_NVAL|ERTS_POLL_EV_ERR)) + +#define ev2str(ev) \ + (((ev) == 0 || (ev) == ERTS_POLL_EV_NONE) ? "NONE" : \ + ((ev) == ERTS_POLL_EV_IN ? "IN" : \ + ((ev) == ERTS_POLL_EV_OUT ? "OUT" : \ + ((ev) == (ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT) ? "IN|OUT" : \ + ((ev) & ERTS_POLL_EV_ERR ? "ERR" : \ + ((ev) & ERTS_POLL_EV_NVAL ? "NVAL" : "OTHER")))))) + + +typedef struct ERTS_POLL_EXPORT(erts_pollset) ErtsPollSet; + typedef struct { char *primary; - char *fallback; char *kernel_poll; Uint memory_size; - int poll_set_size; - int fallback_poll_set_size; + Uint poll_set_size; int lazy_updates; - int pending_updates; + Uint pending_updates; int batch_updates; int concurrent_updates; - int max_fds; -#ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS - long no_avoided_wakeups; - long no_avoided_interrupts; - long no_interrupt_timed; -#endif + int is_fallback; + Uint max_fds; + Uint active_fds; + Uint poll_threads; } ErtsPollInfo; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -void ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet); +#if defined(ERTS_POLL_USE_FALLBACK) && ERTS_KERNEL_POLL_VERSION +# undef ERTS_POLL_EXPORT +# define ERTS_POLL_EXPORT(FUNC) FUNC ## _flbk +# include "erl_poll_api.h" +# undef ERTS_POLL_EXPORT +# define ERTS_POLL_EXPORT(FUNC) FUNC +#elif !defined(ERTS_POLL_USE_FALLBACK) +# define ERTS_POLL_USE_FALLBACK 0 #endif -void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet, - int); -void ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet, - int, - ErtsMonotonicTime); -ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet, - ErtsSysFdType, - ErtsPollEvents, - int on, - int* wake_poller - ); -void ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet, - ErtsPollControlEntry [], - int on); -int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet, - ErtsPollResFd [], - int *, - ErtsMonotonicTime); -int ERTS_POLL_EXPORT(erts_poll_max_fds)(void); -void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet, - ErtsPollInfo *); -ErtsPollSet ERTS_POLL_EXPORT(erts_poll_create_pollset)(void); -void ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet); -void ERTS_POLL_EXPORT(erts_poll_init)(void); -void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet, - ErtsPollEvents [], - int); +#include "erl_poll_api.h" + +/** + * Get the next size of the array that holds the file descriptors. + * This function is used in order for the check io array and the + * pollset array to be of the same size. + */ int erts_poll_new_table_len(int old_len, int need_len); #endif /* #ifndef ERL_POLL_H__ */ diff --git a/erts/emulator/sys/common/erl_poll_api.h b/erts/emulator/sys/common/erl_poll_api.h new file mode 100644 index 0000000000..04beb37d1c --- /dev/null +++ b/erts/emulator/sys/common/erl_poll_api.h @@ -0,0 +1,122 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/** + * @description: Poll interface functions + * @author Lukas Larsson + * + * The functions in the header are used to interact with the poll + * implementation. Iff the kernel-poll implementation needs a fallback + * pollset, then all functions are exported twice. Once with a _flbk + * suffix and once without any suffix. If no fallback is needed, then + * only the non-suffix version is exported. + */ + +/** + * Initialize the poll implementation. Has to be called before any other function. + * @param[out] concurrent_waiters if not NULL, set to 1 if more then one thread + * is allowed to wait in the pollsets at the same time. + */ +void ERTS_POLL_EXPORT(erts_poll_init)(int *concurrent_waiters); +/** + * @brief Create a new pollset. + * @param id The unique debug id of this pollset. + */ +ErtsPollSet *ERTS_POLL_EXPORT(erts_poll_create_pollset)(int id); + +/** + * Modify the contents of a pollset. This function can be called while one + * (or possibly more) thread is waiting in the pollset. + * + * @param ps the pollset to modify + * @param fd the file descriptor to modify + * @param op the type of operation to do. Normal usage is ADD,MOD...MOD,DEL. + * @param evts the events that we are changing interest to. Ignored if op is DEL. + * @param[in] wake_poller if set to 1 any thread waiting in the pollset will be woken. + * This parameter is ignored if the pollset supports concurrent waiters. + * @param[out] wake_poller set to 1 if the waiting thread was woken. + * @return The events set, or ERTS_POLL_EV_NVAL if it was not possible to add the + * fd to the pollset. + */ +ErtsPollEvents ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet *ps, + ErtsSysFdType fd, + ErtsPollOp op, + ErtsPollEvents evts, + int *wake_poller); + +/** + * Wait for events to be ready in the pollset. If the erts_poll_init call + * set concurrent_waiters to 1, then multiple threads are allowed to call + * this function at the same time. + * + * When an event has been triggered on a fd, that event is disabled. To + * re-enable it the implementation has to call erts_poll_control again. + * + * @param ps the pollset to wait for events in + * @param res an array of fd results that the ready fds are put in. + * @param[in] length the length of the res array + * @param[out] length the number of ready events returned in res + * @return 0 on success, else the ERRNO of the error that happened. + */ +int ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet *ps, + ErtsPollResFd res[], + int *length); +/** + * Interrupt the thread waiting in the pollset. This function should be called + * with set = 0 before any thread calls erts_poll_wait in order to clear any + * interrupts that have happened while the thread was awake. + * + * This function has no effect on pollsets that support concurrent waiters. + * + * @param ps the pollset to wake + * @param set if 1, interrupt the pollset, if 0 clear the interrupt flag. + */ +void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet *ps, int set); + +/* Debug functions */ + +/** + * Get the maximum number of fds supported by the pollset + */ +int ERTS_POLL_EXPORT(erts_poll_max_fds)(void); +/** + * Get information about the given pollset + */ +void ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet *ps, + ErtsPollInfo *info); +/** + * Get information about which events are currently selected. + * + * The unix fd is used to index into the array, so naturally this function does + * not work on windows. If the pollset cannot figure out what the selected + * events for a given fd is, it is set to ERTS_POLL_EV_NONE. + * + * @param ps the pollset to get events from + * @param evts an array of which events are selected on. + */ +void ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet *ps, + ErtsPollEvents evts[], + int length); + +#ifdef ERTS_ENABLE_LOCK_COUNT +/** + * Enable lock counting of any locks within the pollset. + */ +void ERTS_POLL_EXPORT(erts_lcnt_enable_pollset_lock_count)(ErtsPollSet *, int enable); +#endif diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c index 79f87eb3a9..420138ff0a 100644 --- a/erts/emulator/sys/common/erl_sys_common_misc.c +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -45,14 +45,6 @@ #endif #endif -/* - * erts_check_io_time is used by the erl_check_io implementation. The - * global erts_check_io_time variable is declared here since there - * (often) exist two versions of erl_check_io (kernel-poll and - * non-kernel-poll), and we dont want two versions of this variable. - */ -erts_smp_atomic_t erts_check_io_time; - /* Written once and only once */ static int filename_encoding = ERL_FILENAME_UNKNOWN; diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index 22059d21d5..b6f5b319ee 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -86,6 +86,10 @@ #include <sys/times.h> +#ifdef HAVE_SYS_RESOURCE_H +# include <sys/resource.h> +#endif + #ifdef HAVE_IEEEFP_H #include <ieeefp.h> #endif @@ -128,10 +132,6 @@ /* File descriptors are numbers anc consecutively allocated on Unix */ #define ERTS_SYS_CONTINOUS_FD_NUMBERS -#ifndef ERTS_SMP -# undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -# define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#endif typedef void *GETENV_STATE; @@ -354,9 +354,7 @@ extern void erts_sys_unix_later_init(void); #ifdef NO_FPE_SIGNALS #define erts_get_current_fp_exception() NULL -#ifdef ERTS_SMP #define erts_thread_init_fp_exception() do{}while(0) -#endif # define __ERTS_FP_CHECK_INIT(fpexnp) do {} while (0) # define __ERTS_FP_ERROR(fpexnp, f, Action) if (!isfinite(f)) { Action; } else {} # define __ERTS_FP_ERROR_THOROUGH(fpexnp, f, Action) __ERTS_FP_ERROR(fpexnp, f, Action) @@ -369,9 +367,7 @@ extern void erts_sys_unix_later_init(void); #else /* !NO_FPE_SIGNALS */ extern volatile unsigned long *erts_get_current_fp_exception(void); -#ifdef ERTS_SMP extern void erts_thread_init_fp_exception(void); -#endif # if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) # define erts_fwait(fpexnp,f) \ __asm__ __volatile__("fwait" : "=m"(*(fpexnp)) : "m"(f)) @@ -438,10 +434,8 @@ void erts_sys_unblock_fpe(int); /* Threads */ -#ifdef USE_THREADS extern int init_async(int); extern int exit_async(void); -#endif #define ERTS_EXIT_AFTER_DUMP _exit diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 5cf0a49972..6315135151 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -58,14 +58,12 @@ #define __DARWIN__ 1 #endif -#ifdef USE_THREADS #include "erl_threads.h" -#endif #include "erl_mseg.h" extern char **environ; -erts_smp_rwmtx_t environ_rwmtx; +erts_rwmtx_t environ_rwmtx; #define MAX_VSIZE 16 /* Max number of entries allowed in an I/O * vector sock_sendv(). @@ -94,19 +92,12 @@ extern void erts_sys_init_float(void); static int debug_log = 0; #endif -#ifdef ERTS_SMP -static erts_smp_atomic32_t have_prepared_crash_dump; -#define ERTS_PREPARED_CRASH_DUMP \ - ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) -#else -static volatile int have_prepared_crash_dump; +static erts_atomic32_t have_prepared_crash_dump; #define ERTS_PREPARED_CRASH_DUMP \ - (have_prepared_crash_dump++) -#endif + ((int) erts_atomic32_xchg_nob(&have_prepared_crash_dump, 1)) -erts_smp_atomic_t sys_misc_mem_sz; +erts_atomic_t sys_misc_mem_sz; -#if defined(ERTS_SMP) static void smp_sig_notify(int signum); static int sig_notify_fds[2] = {-1, -1}; @@ -114,7 +105,6 @@ static int sig_notify_fds[2] = {-1, -1}; static int sig_suspend_fds[2] = {-1, -1}; #endif -#endif jmp_buf erts_sys_sigsegv_jmp; @@ -128,38 +118,12 @@ static int max_files = -1; /* * a few variables used by the break handler */ -#ifdef ERTS_SMP -erts_smp_atomic32_t erts_break_requested; +erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -volatile int erts_break_requested = 0; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#ifndef ERTS_SMP -static Eterm signalstate_sigterm[] = { - am_sigint, /* 0 */ - am_sighup, /* 1 */ - am_sigquit, /* 2 */ - am_sigabrt, /* 3 */ - am_sigalrm, /* 4 */ - am_sigterm, /* 5 */ - am_sigusr1, /* 6 */ - am_sigusr2, /* 7 */ - am_sigchld, /* 8 */ - am_sigstop, /* 9 */ - am_sigtstp /* 10 */ -}; - -volatile Uint erts_signal_state = 0; -#define ERTS_SET_SIGNAL_STATE(S) (erts_signal_state |= signum_to_signalstate(S)) -#define ERTS_CLEAR_SIGNAL_STATE (erts_signal_state = 0) -static ERTS_INLINE Uint signum_to_signalstate(int signum); -#endif /* set early so the break handler has access to initial mode */ static struct termios initial_tty_mode; @@ -167,137 +131,6 @@ static int replace_intr = 0; /* assume yes initially, ttsl_init will clear it */ int using_oldshell = 1; -#ifdef ERTS_ENABLE_KERNEL_POLL - -int erts_use_kernel_poll = 0; - -struct { - int (*select)(ErlDrvPort, ErlDrvEvent, int, int); - int (*enif_select)(ErlNifEnv*, ErlNifEvent, enum ErlNifSelectFlags, void*, const ErlNifPid*, Eterm); - int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData); - void (*check_io_as_interrupt)(void); - void (*check_io_interrupt)(int); - void (*check_io_interrupt_tmd)(int, ErtsMonotonicTime); - void (*check_io)(int); - Uint (*size)(void); - Eterm (*info)(void *); - int (*check_io_debug)(ErtsCheckIoDebugInfo *); -} io_func = {0}; - - -int -driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on) -{ - return (*io_func.select)(port, event, mode, on); -} - -int -driver_event(ErlDrvPort port, ErlDrvEvent event, ErlDrvEventData event_data) -{ - return (*io_func.event)(port, event, event_data); -} - -int enif_select(ErlNifEnv* env, ErlNifEvent event, - enum ErlNifSelectFlags flags, void* obj, const ErlNifPid* pid, Eterm ref) -{ - return (*io_func.enif_select)(env, event, flags, obj, pid, ref); -} - - -Eterm erts_check_io_info(void *p) -{ - return (*io_func.info)(p); -} - -int -erts_check_io_debug(ErtsCheckIoDebugInfo *ip) -{ - return (*io_func.check_io_debug)(ip); -} - - -static void -init_check_io(void) -{ - if (erts_use_kernel_poll) { - io_func.select = driver_select_kp; - io_func.enif_select = enif_select_kp; - io_func.event = driver_event_kp; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT - io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_kp; -#endif - io_func.check_io_interrupt = erts_check_io_interrupt_kp; - io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_kp; - io_func.check_io = erts_check_io_kp; - io_func.size = erts_check_io_size_kp; - io_func.info = erts_check_io_info_kp; - io_func.check_io_debug = erts_check_io_debug_kp; - erts_init_check_io_kp(); - max_files = erts_check_io_max_files_kp(); - } - else { - io_func.select = driver_select_nkp; - io_func.enif_select = enif_select_nkp; - io_func.event = driver_event_nkp; -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT - io_func.check_io_as_interrupt = erts_check_io_async_sig_interrupt_nkp; -#endif - io_func.check_io_interrupt = erts_check_io_interrupt_nkp; - io_func.check_io_interrupt_tmd = erts_check_io_interrupt_timed_nkp; - io_func.check_io = erts_check_io_nkp; - io_func.size = erts_check_io_size_nkp; - io_func.info = erts_check_io_info_nkp; - io_func.check_io_debug = erts_check_io_debug_nkp; - erts_init_check_io_nkp(); - max_files = erts_check_io_max_files_nkp(); - } -} - -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_as_interrupt)() -#else -#define ERTS_CHK_IO_AS_INTR() (*io_func.check_io_interrupt)(1) -#endif -#define ERTS_CHK_IO_INTR (*io_func.check_io_interrupt) -#define ERTS_CHK_IO_INTR_TMD (*io_func.check_io_interrupt_tmd) -#define ERTS_CHK_IO (*io_func.check_io) -#define ERTS_CHK_IO_SZ (*io_func.size) - -#else /* !ERTS_ENABLE_KERNEL_POLL */ - -static void -init_check_io(void) -{ - erts_init_check_io(); - max_files = erts_check_io_max_files(); -} - -#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT -#define ERTS_CHK_IO_AS_INTR() erts_check_io_async_sig_interrupt() -#else -#define ERTS_CHK_IO_AS_INTR() erts_check_io_interrupt(1) -#endif -#define ERTS_CHK_IO_INTR erts_check_io_interrupt -#define ERTS_CHK_IO_INTR_TMD erts_check_io_interrupt_timed -#define ERTS_CHK_IO erts_check_io -#define ERTS_CHK_IO_SZ erts_check_io_size - -#endif - -void -erts_sys_schedule_interrupt(int set) -{ - ERTS_CHK_IO_INTR(set); -} - -#ifdef ERTS_SMP -void -erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) -{ - ERTS_CHK_IO_INTR_TMD(set, timeout_time); -} -#endif - UWord erts_sys_get_page_size(void) { @@ -313,8 +146,8 @@ erts_sys_get_page_size(void) Uint erts_sys_misc_mem_sz(void) { - Uint res = ERTS_CHK_IO_SZ(); - res += erts_smp_atomic_read_mb(&sys_misc_mem_sz); + Uint res = erts_check_io_size(); + res += erts_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -339,7 +172,6 @@ MALLOC_USE_HASH(1); #endif #endif -#ifdef USE_THREADS #ifdef ERTS_THR_HAVE_SIG_FUNCS @@ -418,19 +250,15 @@ thr_create_prepare_child(void *vtcdp) erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } -#endif /* #ifdef USE_THREADS */ void erts_sys_pre_init(void) { -#ifdef USE_THREADS erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; -#endif erts_printf_add_cr_to_stdout = 1; erts_printf_add_cr_to_stderr = 1; -#ifdef USE_THREADS eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -438,33 +266,29 @@ erts_sys_pre_init(void) /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup, +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); +#endif + erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_post_thr_init(); #endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif -#endif /* USE_THREADS */ erts_init_sys_time_sup(); -#ifdef USE_THREADS -#ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&erts_break_requested, 0); - erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0); -#else - erts_break_requested = 0; - have_prepared_crash_dump = 0; -#endif + erts_atomic32_init_nob(&erts_break_requested, 0); + erts_atomic32_init_nob(&have_prepared_crash_dump, 0); -#endif /* USE_THREADS */ - erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); + erts_atomic_init_nob(&sys_misc_mem_sz, 0); { /* @@ -527,10 +351,8 @@ SIGFUNC sys_signal(int sig, SIGFUNC func) return(oact.sa_handler); } -#ifdef USE_THREADS #undef sigprocmask #define sigprocmask erts_thr_sigmask -#endif void sys_sigblock(int sig) { @@ -668,7 +490,7 @@ static void signal_notify_requested(Eterm type) { erts_queue_message(p, locks, msgp, msg, am_system); if (locks) - erts_smp_proc_unlock(p, locks); + erts_proc_unlock(p, locks); erts_proc_dec_refc(p); } } @@ -681,23 +503,17 @@ break_requested(void) * just set a flag - checked for and handled by * scheduler threads erts_check_io() (not signal handler). */ -#ifdef DEBUG - fprintf(stderr,"break!\n"); -#endif if (ERTS_BREAK_REQUESTED) erts_exit(ERTS_INTR_EXIT, ""); ERTS_SET_BREAK_REQUESTED; - ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ + /* Wake aux thread to get handle break */ + erts_aux_thread_poke(); } static RETSIGTYPE request_break(int signum) { -#ifdef ERTS_SMP smp_sig_notify(signum); -#else - break_requested(); -#endif } #ifdef ETHR_UNUSABLE_SIGUSRX @@ -806,35 +622,9 @@ signum_to_signalterm(int signum) } } -#ifndef ERTS_SMP -static ERTS_INLINE Uint -signum_to_signalstate(int signum) -{ - switch (signum) { - case SIGINT: return (1 << 0); - case SIGHUP: return (1 << 1); - case SIGQUIT: return (1 << 2); - case SIGABRT: return (1 << 3); - case SIGALRM: return (1 << 4); - case SIGTERM: return (1 << 5); - case SIGUSR1: return (1 << 6); - case SIGUSR2: return (1 << 7); - case SIGCHLD: return (1 << 8); - case SIGSTOP: return (1 << 9); - case SIGTSTP: return (1 << 10); - default: return 0; - } -} -#endif - static RETSIGTYPE generic_signal_handler(int signum) { -#ifdef ERTS_SMP smp_sig_notify(signum); -#else - ERTS_SET_SIGNAL_STATE(signum); - ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ -#endif } int erts_set_signal(Eterm signal, Eterm type) { @@ -906,7 +696,7 @@ erts_sys_unix_later_init(void) int sys_max_files(void) { - return(max_files); + return max_files; } /************************** OS info *******************************/ @@ -961,7 +751,7 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) { void init_getenv_state(GETENV_STATE *state) { - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); *state = NULL; } @@ -970,7 +760,7 @@ char *getenv_string(GETENV_STATE *state0) char **state = (char **) *state0; char *cp; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state == NULL) state = environ; @@ -984,7 +774,7 @@ char *getenv_string(GETENV_STATE *state0) void fini_getenv_state(GETENV_STATE *state) { *state = NULL; - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); } void erts_do_break_handling(void) @@ -997,7 +787,7 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); /* during break we revert to initial settings */ /* this is done differently for oldshell */ @@ -1025,25 +815,9 @@ void erts_do_break_handling(void) tcsetattr(0,TCSANOW,&temp_mode); } - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); } -#ifdef ERTS_SIGNAL_STATE -void erts_handle_signal_state(void) { - Uint signal_state = ERTS_SIGNAL_STATE; - Uint i = 0; - - ERTS_CLEAR_SIGNAL_STATE; - - while (signal_state) { - if (signal_state & 0x1) { - signal_notify_requested(signalstate_sigterm[i]); - } - i++; - signal_state = signal_state >> 1; - } -} -#endif /* Fills in the systems representation of the jam/beam process identifier. ** The Pid is put in STRING representation in the supplied buffer, @@ -1071,14 +845,14 @@ erts_sys_putenv(char *key, char *value) env = erts_alloc(ERTS_ALC_T_TMP, need); #else env = erts_alloc(ERTS_ALC_T_PUTENV_STR, need); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, need); + erts_atomic_add_nob(&sys_misc_mem_sz, need); #endif strcpy(env,key); strcat(env,"="); strcat(env,value); - erts_smp_rwmtx_rwlock(&environ_rwmtx); + erts_rwmtx_rwlock(&environ_rwmtx); res = putenv(env); - erts_smp_rwmtx_rwunlock(&environ_rwmtx); + erts_rwmtx_rwunlock(&environ_rwmtx); #ifdef HAVE_COPYING_PUTENV erts_free(ERTS_ALC_T_TMP, env); #endif @@ -1125,9 +899,9 @@ int erts_sys_getenv(char *key, char *value, size_t *size) { int res; - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); res = erts_sys_getenv__(key, value, size); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); return res; } @@ -1135,9 +909,9 @@ int erts_sys_unsetenv(char *key) { int res; - erts_smp_rwmtx_rwlock(&environ_rwmtx); + erts_rwmtx_rwlock(&environ_rwmtx); res = unsetenv(key); - erts_smp_rwmtx_rwunlock(&environ_rwmtx); + erts_rwmtx_rwunlock(&environ_rwmtx); return res; } @@ -1278,16 +1052,6 @@ erl_assert_error(const char* expr, const char* func, const char* file, int line) fprintf(stderr, "%s:%d:%s() Assertion failed: %s\n", file, line, func, expr); fflush(stderr); -#if !defined(ERTS_SMP) && 0 - /* Writing a crashdump from a failed assertion when smp support - * is enabled almost a guaranteed deadlocking, don't even bother. - * - * It could maybe be useful (but I'm not convinced) to write the - * crashdump if smp support is disabled... - */ - if (erts_initialized) - erl_crash_dump(file, line, "Assertion failed: %s\n", expr); -#endif abort(); } @@ -1309,22 +1073,7 @@ erl_debug(char* fmt, ...) #endif /* DEBUG */ -/* - * Called from schedule() when it runs out of runnable processes, - * or when Erlang code has performed INPUT_REDUCTIONS reduction - * steps. runnable == 0 iff there are no runnable Erlang processes. - */ -void -erl_sys_schedule(int runnable) -{ - ERTS_CHK_IO(!runnable); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); -} - - -#ifdef ERTS_SMP - -static erts_smp_tid_t sig_dispatcher_tid; +static erts_tid_t sig_dispatcher_tid; static void smp_sig_notify(int signum) @@ -1398,7 +1147,7 @@ signal_dispatcher_thread_func(void *unused) } signal_notify_requested(signal); } - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + ERTS_LC_ASSERT(!erts_thr_progress_is_blocking()); } return NULL; } @@ -1406,7 +1155,7 @@ signal_dispatcher_thread_func(void *unused) static void init_smp_sig_notify(void) { - erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; thr_opts.detached = 1; thr_opts.name = "sys_sig_dispatcher"; @@ -1418,7 +1167,7 @@ init_smp_sig_notify(void) } /* Start signal handler thread */ - erts_smp_thr_create(&sig_dispatcher_tid, + erts_thr_create(&sig_dispatcher_tid, signal_dispatcher_thread_func, NULL, &thr_opts); @@ -1511,102 +1260,19 @@ erts_sys_main_thread(void) } } -#endif /* ERTS_SMP */ - -#ifdef ERTS_ENABLE_KERNEL_POLL /* get_value() is currently only used when - kernel-poll is enabled */ - -/* Get arg marks argument as handled by - putting NULL in argv */ -static char * -get_value(char* rest, char** argv, int* ip) -{ - char *param = argv[*ip]+1; - argv[*ip] = NULL; - if (*rest == '\0') { - char *next = argv[*ip + 1]; - if (next[0] == '-' - && next[1] == '-' - && next[2] == '\0') { - erts_fprintf(stderr, "bad \"%s\" value: \n", param); - erts_usage(); - } - (*ip)++; - argv[*ip] = NULL; - return next; - } - return rest; -} - -#endif /* ERTS_ENABLE_KERNEL_POLL */ void erl_sys_args(int* argc, char** argv) { - int i, j; - - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); - i = 1; + erts_rwmtx_init(&environ_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); ASSERT(argc && argv); - while (i < *argc) { - if(argv[i][0] == '-') { - switch (argv[i][1]) { -#ifdef ERTS_ENABLE_KERNEL_POLL - case 'K': { - char *arg = get_value(argv[i] + 2, argv, &i); - if (strcmp("true", arg) == 0) { - erts_use_kernel_poll = 1; - } - else if (strcmp("false", arg) == 0) { - erts_use_kernel_poll = 0; - } - else { - erts_fprintf(stderr, "bad \"K\" value: %s\n", arg); - erts_usage(); - } - break; - } -#endif - case '-': - goto done_parsing; - default: - break; - } - } - i++; - } - - done_parsing: - -#ifdef ERTS_ENABLE_KERNEL_POLL - if (erts_use_kernel_poll) { - char no_kp[10]; - size_t no_kp_sz = sizeof(no_kp); - int res = erts_sys_getenv_raw("ERL_NO_KERNEL_POLL", no_kp, &no_kp_sz); - if (res > 0 - || (res == 0 - && sys_strcmp("false", no_kp) != 0 - && sys_strcmp("FALSE", no_kp) != 0)) { - erts_use_kernel_poll = 0; - } - } -#endif - - init_check_io(); + max_files = erts_check_io_max_files(); -#ifdef ERTS_SMP init_smp_sig_notify(); init_smp_sig_suspend(); -#endif - /* Handled arguments have been marked with NULL. Slide arguments - not handled towards the beginning of argv. */ - for (i = 0, j = 0; i < *argc; i++) { - if (argv[i]) - argv[j++] = argv[i]; - } - *argc = j; } diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 834706d86f..0228e1af54 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -53,14 +53,12 @@ #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" -#ifdef USE_THREADS #include "erl_threads.h" -#endif extern char **environ; -extern erts_smp_rwmtx_t environ_rwmtx; +extern erts_rwmtx_t environ_rwmtx; -extern erts_smp_atomic_t sys_misc_mem_sz; +extern erts_atomic_t sys_misc_mem_sz; static Eterm forker_port; @@ -86,12 +84,6 @@ static Eterm forker_port; #define MAXIOV 16 #endif -#ifdef USE_THREADS -# define FDBLOCK 1 -#else -# define FDBLOCK 0 -#endif - /* Used by the fd driver iff the fd could not be set to non-blocking */ typedef struct ErtsSysBlocking_ { ErlDrvPDL pdl; @@ -178,9 +170,7 @@ void erl_sys_late_init(void) { SysDriverOpts opts; -#ifdef ERTS_SMP Port *port; -#endif sys_signal(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */ @@ -197,13 +187,9 @@ erl_sys_late_init(void) opts.argv = NULL; opts.parallelism = erts_port_parallelism; -#ifdef ERTS_SMP port = -#endif erts_open_driver(&forker_driver, make_internal_pid(0), "forker", &opts, NULL, NULL); -#ifdef ERTS_SMP erts_mtx_unlock(port->lock); -#endif erts_sys_unix_later_init(); /* Need to be called after forker has been started */ } @@ -220,10 +206,8 @@ static ErlDrvData vanilla_start(ErlDrvPort, char*, SysDriverOpts*); /* II.III FD prototypes */ static ErlDrvData fd_start(ErlDrvPort, char*, SysDriverOpts*); -#if FDBLOCK static void fd_async(void *); static void fd_ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data); -#endif static ErlDrvSSizeT fd_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT, char **, ErlDrvSizeT); static void fd_stop(ErlDrvData); @@ -287,11 +271,7 @@ struct erl_drv_entry fd_driver_entry = { fd_control, NULL, outputv, -#if FDBLOCK fd_ready_async, /* ready_async */ -#else - NULL, -#endif fd_flush, /* flush */ NULL, /* call */ NULL, /* event */ @@ -363,7 +343,7 @@ static int set_blocking_data(ErtsSysDriverData *dd) { dd->blocking = erts_alloc(ERTS_ALC_T_SYS_BLOCKING, sizeof(ErtsSysBlocking)); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking)); + erts_atomic_add_nob(&sys_misc_mem_sz, sizeof(ErtsSysBlocking)); dd->blocking->pdl = driver_pdl_create(dd->port_num); dd->blocking->res = 0; @@ -406,7 +386,7 @@ create_driver_data(ErlDrvPort port_num, size += sizeof(ErtsSysFdData); data = erts_alloc(ERTS_ALC_T_DRV_TAB,size); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, size); + erts_atomic_add_nob(&sys_misc_mem_sz, size); driver_data = (ErtsSysDriverData*)data; data += sizeof(*driver_data); @@ -441,7 +421,7 @@ create_driver_data(ErlDrvPort port_num, data += sizeof(*driver_data->ofd); init_fd_data(driver_data->ofd, ofd); } - if (is_blocking && FDBLOCK) + if (is_blocking) if (!set_blocking_data(driver_data)) { erts_free(ERTS_ALC_T_DRV_TAB, driver_data); return NULL; @@ -472,7 +452,7 @@ static char **build_unix_environment(char *block) char **cpp; char** old_env; - ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx)); + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); cp = block; len = 0; @@ -620,12 +600,12 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, len = CMD_LINE_PREFIX_STR_SZ + len + 1; } - erts_smp_rwmtx_rlock(&environ_rwmtx); + erts_rwmtx_rlock(&environ_rwmtx); if (opts->envir == NULL) { new_environ = environ; } else if ((new_environ = build_unix_environment(opts->envir)) == NULL) { - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); close_pipes(ifd, ofd); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); errno = ENOMEM; @@ -641,7 +621,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); errno = err; return ERL_DRV_ERROR_ERRNO; } @@ -681,7 +661,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (!io_vector) { close_pipes(ifd, ofd); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); @@ -766,7 +746,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, erts_free(ERTS_ALC_T_TMP, io_vector); if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); erts_free(ERTS_ALC_T_TMP, (void *) cmd_line); errno = err; return ERL_DRV_ERROR_ERRNO; @@ -795,7 +775,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, if (new_environ != environ) erts_free(ERTS_ALC_T_ENVIRONMENT, (void *) new_environ); - erts_smp_rwmtx_runlock(&environ_rwmtx); + erts_rwmtx_runlock(&environ_rwmtx); dd = create_driver_data(port_num, ifd[0], ofd[1], opts->packet_bytes, DO_WRITE | DO_READ, opts->exit_status, @@ -1068,8 +1048,8 @@ static void clear_fd_data(ErtsSysFdData *fdd) { if (fdd->sz > 0) { erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fdd->buf); - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= fdd->sz); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*fdd->sz); } fdd->buf = NULL; fdd->sz = 0; @@ -1092,13 +1072,11 @@ static void fd_stop(ErlDrvData ev) /* Does not close the fds */ ErlDrvPort prt = dd->port_num; int sz = sizeof(ErtsSysDriverData); -#if FDBLOCK if (dd->blocking) { erts_free(ERTS_ALC_T_SYS_BLOCKING, dd->blocking); dd->blocking = NULL; sz += sizeof(ErtsSysBlocking); } -#endif if (dd->ifd) { sz += sizeof(ErtsSysFdData); @@ -1110,7 +1088,7 @@ static void fd_stop(ErlDrvData ev) /* Does not close the fds */ } erts_free(ERTS_ALC_T_DRV_TAB, dd); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sz); + erts_atomic_add_nob(&sys_misc_mem_sz, -sz); } static void fd_flush(ErlDrvData ev) @@ -1191,19 +1169,19 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) ev->iov[0].iov_len = pb; ev->size += pb; - if (dd->blocking && FDBLOCK) + if (dd->blocking) driver_pdl_lock(dd->blocking->pdl); if ((sz = driver_sizeq(ix)) > 0) { driver_enqv(ix, ev, 0); - if (dd->blocking && FDBLOCK) + if (dd->blocking) driver_pdl_unlock(dd->blocking->pdl); if (sz + ev->size >= (1 << 13)) set_busy_port(ix, 1); } - else if (!dd->blocking || !FDBLOCK) { + else if (!dd->blocking) { /* We try to write directly if the fd in non-blocking */ int vsize = ev->vsize > MAX_VSIZE ? MAX_VSIZE : ev->vsize; @@ -1220,7 +1198,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) driver_enqv(ix, ev, n); /* n is the skip value */ driver_select(ix, ofd, ERL_DRV_WRITE|ERL_DRV_USE, 1); } -#if FDBLOCK else { if (ev->size != 0) { driver_enqv(ix, ev, 0); @@ -1231,7 +1208,6 @@ static void outputv(ErlDrvData e, ErlIOVec* ev) driver_pdl_unlock(dd->blocking->pdl); } } -#endif /* return 0;*/ } @@ -1303,7 +1279,7 @@ static int port_inp_failure(ErtsSysDriverData *dd, int res) clear_fd_data(dd->ifd); } - if (dd->blocking && FDBLOCK) { + if (dd->blocking) { driver_pdl_lock(dd->blocking->pdl); if (driver_sizeq(dd->port_num) > 0) { driver_pdl_unlock(dd->blocking->pdl); @@ -1408,7 +1384,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) if (dd->ifd->fd < 0) { driver_select(port_num, abs(dd->ifd->fd), ERL_DRV_READ|ERL_DRV_USE, 0); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); + erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); dd->ifd = NULL; } @@ -1514,7 +1490,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) port_inp_failure(dd, -1); } else { - erts_smp_atomic_add_nob(&sys_misc_mem_sz, h); + erts_atomic_add_nob(&sys_misc_mem_sz, h); sys_memcpy(buf, cpos, bytes_left); dd->ifd->buf = buf; dd->ifd->sz = h; @@ -1549,7 +1525,7 @@ static void ready_output(ErlDrvData e, ErlDrvEvent ready_fd) should close the output fd as soon as the command has been sent. */ driver_select(ix, ready_fd, ERL_DRV_WRITE|ERL_DRV_USE, 0); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); + erts_atomic_add_nob(&sys_misc_mem_sz, -sizeof(ErtsSysFdData)); dd->ofd = NULL; } if (dd->terminating) @@ -1579,7 +1555,6 @@ static void stop_select(ErlDrvEvent fd, void* _) close((int)fd); } -#if FDBLOCK static void fd_async(void *async_data) @@ -1658,7 +1633,6 @@ void fd_ready_async(ErlDrvData drv_data, return; /* 0; */ } -#endif /* Forker driver */ @@ -1749,8 +1723,6 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name, SET_NONBLOCKING(forker_fd); - driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1); - return (ErlDrvData)port_num; } @@ -1847,10 +1819,19 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd) static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { + static int first_call = 1; ErtsSysForkerProto *proto = (ErtsSysForkerProto *)buf; ErlDrvPort port_num = (ErlDrvPort)e; int res; + if (first_call) { + /* + * Do driver_select here when schedulers and their pollsets have started. + */ + driver_select(port_num, forker_fd, ERL_DRV_READ|ERL_DRV_USE, 1); + first_call = 0; + } + driver_enq(port_num, buf, len); if (driver_sizeq(port_num) > sizeof(*proto)) { return 0; diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index 6435da086f..a82c15bd32 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -39,7 +39,6 @@ erts_sys_init_float(void) #else /* !NO_FPE_SIGNALS */ -#ifdef ERTS_SMP static erts_tsd_key_t fpe_key; /* once-only initialisation early in the main thread (via erts_sys_init_float()) */ @@ -61,11 +60,6 @@ static ERTS_INLINE volatile unsigned long *erts_thread_get_fp_exception(void) { return (volatile unsigned long*)erts_tsd_get(fpe_key); } -#else /* !SMP */ -#define erts_init_fp_exception() /*empty*/ -static volatile unsigned long fp_exception; -#define erts_thread_get_fp_exception() (&fp_exception) -#endif /* SMP */ volatile unsigned long *erts_get_current_fp_exception(void) { @@ -659,11 +653,9 @@ void erts_sys_init_float(void) void erts_thread_init_float(void) { -#ifdef ERTS_SMP /* This allows Erlang schedulers to leave Erlang-process context and still have working FP exceptions. XXX: is this needed? */ erts_thread_init_fp_exception(); -#endif #ifndef NO_FPE_SIGNALS /* NOTE: diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c index 4f26639703..ef05380d17 100644 --- a/erts/emulator/sys/unix/sys_time.c +++ b/erts/emulator/sys/unix/sys_time.c @@ -160,7 +160,7 @@ struct sys_time_internal_state_read_mostly__ { #ifdef ERTS_SYS_TIME_INTERNAL_STATE_WRITE_FREQ__ struct sys_time_internal_state_write_freq__ { - erts_smp_mtx_t mtx; + erts_mtx_t mtx; #if defined(__linux__) && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME) ErtsMonotonicTime last_delivered; #endif @@ -304,8 +304,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp) erts_sys_time_data__.r.o.os_times = clock_gettime_times_verified; #endif - erts_smp_mtx_init(&internal_state.w.f.mtx, - "os_monotonic_time"); + erts_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); internal_state.w.f.last_delivered = clock_gettime_monotonic(); init_resp->os_monotonic_time_info.locked_use = 1; @@ -525,12 +525,12 @@ static ErtsMonotonicTime clock_gettime_monotonic_verified(void) mtime = (ErtsMonotonicTime) posix_clock_gettime(MONOTONIC_CLOCK_ID, MONOTONIC_CLOCK_ID_STR); - erts_smp_mtx_lock(&internal_state.w.f.mtx); + erts_mtx_lock(&internal_state.w.f.mtx); if (mtime < internal_state.w.f.last_delivered) mtime = internal_state.w.f.last_delivered; else internal_state.w.f.last_delivered = mtime; - erts_smp_mtx_unlock(&internal_state.w.f.mtx); + erts_mtx_unlock(&internal_state.w.f.mtx); return mtime; } @@ -547,12 +547,12 @@ static void clock_gettime_times_verified(ErtsMonotonicTime *mtimep, WALL_CLOCK_ID_STR, stimep); - erts_smp_mtx_lock(&internal_state.w.f.mtx); + erts_mtx_lock(&internal_state.w.f.mtx); if (*mtimep < internal_state.w.f.last_delivered) *mtimep = internal_state.w.f.last_delivered; else internal_state.w.f.last_delivered = *mtimep; - erts_smp_mtx_unlock(&internal_state.w.f.mtx); + erts_mtx_unlock(&internal_state.w.f.mtx); } #endif /* defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) */ @@ -878,8 +878,6 @@ ErtsMonotonicTime erts_os_monotonic_time(void) { Uint32 ticks = get_tick_count(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); return ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks) << internal_state.r.o.times_shift; } diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index b10fc1e430..fd4c745c3b 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -34,6 +34,7 @@ */ /*#define HARDDEBUG */ +/*#define HARDTRACE */ #ifdef HARDDEBUG #ifdef HARDTRACE #define HARDTRACEF(X) my_debug_printf##X @@ -50,7 +51,7 @@ static void my_debug_printf(char *fmt, ...) va_start(args, fmt); erts_vsnprintf(buffer,1024,fmt,args); va_end(args); - erts_fprintf(stderr,"%s\r\n",buffer); + erts_printf("%s\r\n",buffer); } #else #define HARDTRACEF(X) @@ -142,7 +143,8 @@ static erts_mtx_t save_ops_mtx; static void poll_debug_init(void) { - erts_mtx_init(&save_ops_mtx, "save_ops_lock"); + erts_mtx_init(&save_ops_mtx, "save_ops_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG); } void poll_debug_set_active_fd(ErtsSysFdType fd) @@ -273,53 +275,35 @@ typedef struct _Waiter { /* * The structure for a pollset. There can currently be only one... */ -struct ErtsPollSet_ { +struct erts_pollset { Waiter** waiter; int allocated_waiters; /* Size ow waiter array */ int num_waiters; /* Number of waiter threads. */ - int restore_events; /* Tells us to restore waiters events - next time around */ HANDLE event_io_ready; /* To be used when waiting for io */ /* These are used to wait for workers to enter standby */ volatile int standby_wait_counter; /* Number of threads to wait for */ CRITICAL_SECTION standby_crit; /* CS to guard the counter */ - HANDLE standby_wait_event; /* Event signalled when counte == 0 */ + HANDLE standby_wait_event; /* Event signalled when counter == 0 */ erts_atomic32_t wakeup_state; -#ifdef ERTS_SMP - erts_smp_mtx_t mtx; -#endif - erts_atomic64_t timeout_time; + erts_mtx_t mtx; }; -#ifdef ERTS_SMP #define ERTS_POLLSET_LOCK(PS) \ - erts_smp_mtx_lock(&(PS)->mtx) + erts_mtx_lock(&(PS)->mtx) #define ERTS_POLLSET_UNLOCK(PS) \ - erts_smp_mtx_unlock(&(PS)->mtx) - -#else + erts_mtx_unlock(&(PS)->mtx) -#define ERTS_POLLSET_LOCK(PS) -#define ERTS_POLLSET_UNLOCK(PS) - -#endif /* * Communication with sys_interrupt */ -#ifdef ERTS_SMP -extern erts_smp_atomic32_t erts_break_requested; +extern erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -extern volatile int erts_break_requested; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) static erts_mtx_t break_waiter_lock; static HANDLE break_happened_event; @@ -366,39 +350,19 @@ do { \ wait_standby(PS); \ } while(0) -static ERTS_INLINE void -init_timeout_time(ErtsPollSet ps) -{ - erts_atomic64_init_nob(&ps->timeout_time, - (erts_aint64_t) ERTS_MONOTONIC_TIME_MAX); -} - -static ERTS_INLINE void -set_timeout_time(ErtsPollSet ps, ErtsMonotonicTime time) -{ - erts_atomic64_set_relb(&ps->timeout_time, - (erts_aint64_t) time); -} - -static ERTS_INLINE ErtsMonotonicTime -get_timeout_time(ErtsPollSet ps) -{ - return (ErtsMonotonicTime) erts_atomic64_read_acqb(&ps->timeout_time); -} - #define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) 0) #define ERTS_POLL_WOKEN_IO_READY ((erts_aint32_t) 1) #define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) 2) #define ERTS_POLL_WOKEN_TIMEDOUT ((erts_aint32_t) 3) static ERTS_INLINE int -is_io_ready(ErtsPollSet ps) +is_io_ready(ErtsPollSet *ps) { return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; } static ERTS_INLINE void -woke_up(ErtsPollSet ps) +woke_up(ErtsPollSet *ps) { if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, @@ -421,7 +385,7 @@ woke_up(ErtsPollSet ps) } static ERTS_INLINE int -wakeup_cause(ErtsPollSet ps) +wakeup_cause(ErtsPollSet *ps) { int res; erts_aint32_t wakeup_state = erts_atomic32_read_acqb(&ps->wakeup_state); @@ -444,46 +408,8 @@ wakeup_cause(ErtsPollSet ps) return res; } -static ERTS_INLINE DWORD -poll_wait_timeout(ErtsPollSet ps, ErtsMonotonicTime timeout_time) -{ - ErtsMonotonicTime current_time, diff_time, timeout; - - if (timeout_time == ERTS_POLL_NO_TIMEOUT) { - no_timeout: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MIN); - woke_up(ps); - return (DWORD) 0; - } - - current_time = erts_get_monotonic_time(NULL); - diff_time = timeout_time - current_time; - if (diff_time <= 0) - goto no_timeout; - - /* Round up to nearest milli second */ - timeout = (ERTS_MONOTONIC_TO_MSEC(diff_time - 1) + 1); - if (timeout > INT_MAX) - timeout = INT_MAX; /* Also prevents DWORD overflow */ - - set_timeout_time(ps, current_time + ERTS_MSEC_TO_MONOTONIC(timeout)); - - ResetEvent(ps->event_io_ready); - /* - * Since we don't know the internals of ResetEvent() we issue - * a memory barrier as a safety precaution ensuring that - * the load of wakeup_state wont be reordered with stores made - * by ResetEvent(). - */ - ERTS_THR_MEMORY_BARRIER; - if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) - return (DWORD) 0; - - return (DWORD) timeout; -} - static ERTS_INLINE void -wake_poller(ErtsPollSet ps, int io_ready) +wake_poller(ErtsPollSet *ps, int io_ready) { erts_aint32_t wakeup_state; if (io_ready) { @@ -518,13 +444,13 @@ wake_poller(ErtsPollSet ps, int io_ready) } static ERTS_INLINE void -reset_io_ready(ErtsPollSet ps) +reset_io_ready(ErtsPollSet *ps) { erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); } static ERTS_INLINE void -restore_io_ready(ErtsPollSet ps) +restore_io_ready(ErtsPollSet *ps) { erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); } @@ -534,13 +460,13 @@ restore_io_ready(ErtsPollSet ps) * notifying a poller thread about I/O ready. */ static ERTS_INLINE void -notify_io_ready(ErtsPollSet ps) +notify_io_ready(ErtsPollSet *ps) { wake_poller(ps, 1); } static ERTS_INLINE void -reset_interrupt(ErtsPollSet ps) +reset_interrupt(ErtsPollSet *ps) { /* We need to keep io-ready if set */ erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state); @@ -557,12 +483,12 @@ reset_interrupt(ErtsPollSet ps) } static ERTS_INLINE void -set_interrupt(ErtsPollSet ps) +set_interrupt(ErtsPollSet *ps) { wake_poller(ps, 0); } -static void setup_standby_wait(ErtsPollSet ps, int num_threads) +static void setup_standby_wait(ErtsPollSet *ps, int num_threads) { EnterCriticalSection(&(ps->standby_crit)); ps->standby_wait_counter = num_threads; @@ -570,7 +496,7 @@ static void setup_standby_wait(ErtsPollSet ps, int num_threads) LeaveCriticalSection(&(ps->standby_crit)); } -static void signal_standby(ErtsPollSet ps) +static void signal_standby(ErtsPollSet *ps) { EnterCriticalSection(&(ps->standby_crit)); --(ps->standby_wait_counter); @@ -584,7 +510,7 @@ static void signal_standby(ErtsPollSet ps) LeaveCriticalSection(&(ps->standby_crit)); } -static void wait_standby(ErtsPollSet ps) +static void wait_standby(ErtsPollSet *ps) { WaitForSingleObject(ps->standby_wait_event,INFINITE); } @@ -652,7 +578,7 @@ static void consistency_check(Waiter* w) #endif -static void new_waiter(ErtsPollSet ps) +static void new_waiter(ErtsPollSet *ps) { register Waiter* w; DWORD tid; /* Id for thread. */ @@ -677,7 +603,7 @@ static void new_waiter(ErtsPollSet ps) w->active_events = 1; w->highwater = 1; w->total_events = 1; - erts_mtx_init(&w->mtx, "pollwaiter"); + erts_mtx_init(&w->mtx, "pollwaiter", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); /* @@ -746,7 +672,7 @@ static void *break_waiter(void *param) static void *threaded_waiter(void *param) { register Waiter* w = (Waiter *) param; - ErtsPollSet ps = (ErtsPollSet) w->xdata; + ErtsPollSet *ps = (ErtsPollSet*) w->xdata; #ifdef HARD_POLL_DEBUG2 HANDLE oold_fired[64]; int num_oold_fired; @@ -849,9 +775,9 @@ event_happened: ASSERT(i >= WAIT_OBJECT_0+1); i -= WAIT_OBJECT_0; ASSERT(i >= 1); - w->active_events--; HARDDEBUGF(("i = %d, a,h,t = %d,%d,%d",i, w->active_events, w->highwater, w->total_events)); + w->active_events--; #ifdef HARD_POLL_DEBUG2 fired[num_fired++] = w->events[i]; #endif @@ -881,7 +807,7 @@ event_happened: * The actual adding and removing from pollset utilities */ -static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode) +static int set_driver_select(ErtsPollSet *ps, HANDLE event, ErtsPollEvents mode) { int i; int best_waiter = -1; /* The waiter with lowest number of events. */ @@ -971,13 +897,13 @@ static int set_driver_select(ErtsPollSet ps, HANDLE event, ErtsPollEvents mode) #endif erts_mtx_unlock(&w->mtx); START_WAITER(ps,w); - HARDDEBUGF(("add select %d %d %d %d",best_waiter, + HARDDEBUGF(("%d: add select %d %d %d %d", event, best_waiter, w->active_events,w->highwater,w->total_events)); return mode; } -static int cancel_driver_select(ErtsPollSet ps, HANDLE event) +static int cancel_driver_select(ErtsPollSet *ps, HANDLE event) { int i; @@ -1032,7 +958,7 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event) * Interface functions */ -void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */) +void erts_poll_interrupt(ErtsPollSet *ps, int set /* bool */) { HARDTRACEF(("In erts_poll_interrupt(%d)",set)); if (!set) @@ -1042,35 +968,23 @@ void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */) HARDTRACEF(("Out erts_poll_interrupt(%d)",set)); } -void erts_poll_interrupt_timed(ErtsPollSet ps, - int set /* bool */, - ErtsMonotonicTime timeout_time) -{ - HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,timeout_time)); - if (!set) - reset_interrupt(ps); - else if (get_timeout_time(ps) > timeout_time) - set_interrupt(ps); - HARDTRACEF(("Out erts_poll_interrupt_timed")); -} - /* * Windows is special, there is actually only one event type, and * the only difference between ERTS_POLL_EV_IN and ERTS_POLL_EV_OUT * is which driver callback will eventually be called. */ -static ErtsPollEvents do_poll_control(ErtsPollSet ps, - ErtsSysFdType fd, - ErtsPollEvents pe, - int on /* bool */) +static ErtsPollEvents do_poll_control(ErtsPollSet *ps, + ErtsSysFdType fd, + ErtsPollOp op, + ErtsPollEvents pe) { HANDLE event = (HANDLE) fd; ErtsPollEvents mode; ErtsPollEvents result; ASSERT(event != INVALID_HANDLE_VALUE); - if (on) { + if (op != ERTS_POLL_OP_DEL) { if (pe & ERTS_POLL_EV_IN || !(pe & ERTS_POLL_EV_OUT )) { mode = ERTS_POLL_EV_IN; } else { @@ -1083,51 +997,30 @@ static ErtsPollEvents do_poll_control(ErtsPollSet ps, return result; } -ErtsPollEvents erts_poll_control(ErtsPollSet ps, +ErtsPollEvents erts_poll_control(ErtsPollSet *ps, ErtsSysFdType fd, + ErtsPollOp op, ErtsPollEvents pe, - int on, int* do_wake) /* In: Wake up polling thread */ /* Out: Poller is woken */ { ErtsPollEvents result; - HARDTRACEF(("In erts_poll_control(0x%08X, %u, %d)",(unsigned long) fd, (unsigned) pe, on)); + HARDTRACEF(("In erts_poll_control(0x%08X, %s, %s)", + (unsigned long) fd, op2str(op), ev2str(pe))); ERTS_POLLSET_LOCK(ps); - result=do_poll_control(ps,fd,pe,on); + result=do_poll_control(ps, fd, op, pe); ERTS_POLLSET_UNLOCK(ps); *do_wake = 0; /* Never any need to wake polling threads on windows */ HARDTRACEF(("Out erts_poll_control -> %u",(unsigned) result)); return result; } -void erts_poll_controlv(ErtsPollSet ps, - ErtsPollControlEntry pcev[], - int len) -{ - int i; - int hshur = 0; - int do_wake = 0; - - HARDTRACEF(("In erts_poll_controlv(%d)",len)); - ERTS_POLLSET_LOCK(ps); - - for (i = 0; i < len; i++) { - pcev[i].events = do_poll_control(ps, - pcev[i].fd, - pcev[i].events, - pcev[i].on); - } - ERTS_POLLSET_UNLOCK(ps); - HARDTRACEF(("Out erts_poll_controlv")); -} - -int erts_poll_wait(ErtsPollSet ps, +int erts_poll_wait(ErtsPollSet *ps, ErtsPollResFd pr[], - int *len, - ErtsMonotonicTime timeout_time) + int *len) { int no_fds; - DWORD timeout; + DWORD timeout = INFINITE; EventData* ev; int res = 0; int num = 0; @@ -1138,42 +1031,6 @@ int erts_poll_wait(ErtsPollSet ps, HARDTRACEF(("In erts_poll_wait")); ERTS_POLLSET_LOCK(ps); - if (!is_io_ready(ps) && ps->restore_events) { - HARDDEBUGF(("Restore events: %d",ps->num_waiters)); - ps->restore_events = 0; - for (i = 0; i < ps->num_waiters; ++i) { - Waiter* w = ps->waiter[i]; - erts_mtx_lock(&w->mtx); - HARDDEBUGF(("Maybe reset %d %d %d %d",i, - w->active_events,w->highwater,w->total_events)); - if (w->active_events < w->total_events) { - erts_mtx_unlock(&w->mtx); - STOP_WAITER(ps,w); - HARDDEBUGF(("Need reset %d %d %d %d",i, - w->active_events,w->highwater,w->total_events)); - erts_mtx_lock(&w->mtx); - /* Need reset, just check that it doesn't have got more to tell */ - if (w->highwater != w->active_events) { - HARDDEBUGF(("Oups!")); - /* Oups, got signalled before we took the lock, can't reset */ - if(!is_io_ready(ps)) { - erts_exit(ERTS_ERROR_EXIT,"Internal error: " - "Inconsistent io structures in erl_poll.\n"); - } - START_WAITER(ps,w); - erts_mtx_unlock(&w->mtx); - ps->restore_events = 1; - continue; - } - w->active_events = w->highwater = w->total_events; - START_WAITER(ps,w); - erts_mtx_unlock(&w->mtx); - } else { - erts_mtx_unlock(&w->mtx); - } - } - } - no_fds = *len; #ifdef ERTS_POLL_MAX_RES @@ -1181,26 +1038,29 @@ int erts_poll_wait(ErtsPollSet ps, no_fds = ERTS_POLL_MAX_RES; #endif - timeout = poll_wait_timeout(ps, timeout_time); - - /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/ + ResetEvent(ps->event_io_ready); + /* + * Since we don't know the internals of ResetEvent() we issue + * a memory barrier as a safety precaution ensuring that + * the load of wakeup_state wont be reordered with stores made + * by ResetEvent(). + */ + ERTS_THR_MEMORY_BARRIER; + if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) + timeout = (DWORD) 0; - if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) { + if (!erts_atomic32_read_nob(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; int num_h = 2; - ERTS_MSACC_PUSH_STATE_M(); + ERTS_MSACC_PUSH_STATE(); HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP erts_thr_progress_prepare_wait(NULL); -#endif - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); WaitForMultipleObjects(num_h, harr, FALSE, timeout); -#ifdef ERTS_SMP erts_thr_progress_finalize_wait(NULL); -#endif - ERTS_MSACC_POP_STATE_M(); + ERTS_MSACC_POP_STATE(); ERTS_POLLSET_LOCK(ps); HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); woke_up(ps); @@ -1233,7 +1093,7 @@ int erts_poll_wait(ErtsPollSet ps, reset_io_ready(ps); - n = ps->num_waiters; + n = ps->num_waiters; for (i = 0; i < n; i++) { Waiter* w = ps->waiter[i]; @@ -1259,11 +1119,10 @@ int erts_poll_wait(ErtsPollSet ps, HARDDEBUGF(("To many FD's to report!")); goto done; } - HARDDEBUGF(("SET! Restore events")); - ps->restore_events = 1; HARDDEBUGF(("Report %d,%d",i,j)); - pr[num].fd = (ErtsSysFdType) w->events[j]; - pr[num].events = w->evdata[j]->mode; + ERTS_POLL_RES_SET_FD(&pr[num], w->events[j]); + ERTS_POLL_RES_SET_EVTS(&pr[num], w->evdata[j]->mode); + remove_event_from_set(w, j); #ifdef HARD_POLL_DEBUG poll_debug_reported(w->events[j],w->highwater | (j << 16)); poll_debug_reported(w->events[j],first | (last << 16)); @@ -1271,13 +1130,14 @@ int erts_poll_wait(ErtsPollSet ps, ++num; } + w->total_events = w->highwater = w->active_events; + #ifdef DEBUG consistency_check(w); #endif erts_mtx_unlock(&w->mtx); } done: - set_timeout_time(ps, ERTS_MONOTONIC_TIME_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1292,7 +1152,7 @@ int erts_poll_max_fds(void) return res; } -void erts_poll_info(ErtsPollSet ps, +void erts_poll_info(ErtsPollSet *ps, ErtsPollInfo *pip) { Uint size = 0; @@ -1302,7 +1162,7 @@ void erts_poll_info(ErtsPollSet ps, HARDTRACEF(("In erts_poll_info")); ERTS_POLLSET_LOCK(ps); - size += sizeof(struct ErtsPollSet_); + size += sizeof(struct erts_pollset); size += sizeof(Waiter *) * ps->allocated_waiters; for (i = 0; i < ps->num_waiters; ++i) { Waiter *w = ps->waiter[i]; @@ -1317,16 +1177,12 @@ void erts_poll_info(ErtsPollSet ps, pip->primary = "WaitForMultipleObjects"; - pip->fallback = NULL; - pip->kernel_poll = NULL; pip->memory_size = size; pip->poll_set_size = num_events; - pip->fallback_poll_set_size = 0; - pip->lazy_updates = 0; pip->pending_updates = 0; @@ -1334,6 +1190,8 @@ void erts_poll_info(ErtsPollSet ps, pip->batch_updates = 0; pip->concurrent_updates = 0; + + pip->is_fallback = 0; ERTS_POLLSET_UNLOCK(ps); pip->max_fds = erts_poll_max_fds(); @@ -1341,10 +1199,10 @@ void erts_poll_info(ErtsPollSet ps, } -ErtsPollSet erts_poll_create_pollset(void) +ErtsPollSet *erts_poll_create_pollset(int no) { - ErtsPollSet ps = SEL_ALLOC(ERTS_ALC_T_POLLSET, - sizeof(struct ErtsPollSet_)); + ErtsPollSet *ps = SEL_ALLOC(ERTS_ALC_T_POLLSET, + sizeof(struct erts_pollset)); HARDTRACEF(("In erts_poll_create_pollset")); ps->num_waiters = 0; @@ -1355,19 +1213,15 @@ ErtsPollSet erts_poll_create_pollset(void) ps->standby_wait_counter = 0; ps->event_io_ready = CreateManualEvent(FALSE); ps->standby_wait_event = CreateManualEvent(FALSE); - ps->restore_events = 0; erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); -#ifdef ERTS_SMP - erts_smp_mtx_init(&ps->mtx, "pollset"); -#endif - init_timeout_time(ps); + erts_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; } -void erts_poll_destroy_pollset(ErtsPollSet ps) +void erts_poll_destroy_pollset(ErtsPollSet *ps) { int i; HARDTRACEF(("In erts_poll_destroy_pollset")); @@ -1390,9 +1244,7 @@ void erts_poll_destroy_pollset(ErtsPollSet ps) CloseHandle(ps->event_io_ready); CloseHandle(ps->standby_wait_event); ERTS_POLLSET_UNLOCK(ps); -#ifdef ERTS_SMP - erts_smp_mtx_destroy(&ps->mtx); -#endif + erts_mtx_destroy(&ps->mtx); SEL_FREE(ERTS_ALC_T_POLLSET, (void *) ps); HARDTRACEF(("Out erts_poll_destroy_pollset")); } @@ -1400,36 +1252,44 @@ void erts_poll_destroy_pollset(ErtsPollSet ps) /* * Actually mostly initializes the friend module sys_interrupt... */ -void erts_poll_init(void) +void erts_poll_init(int *concurrent_updates) { - erts_tid_t thread; #ifdef HARD_POLL_DEBUG poll_debug_init(); #endif + if (concurrent_updates) + *concurrent_updates = 0; + HARDTRACEF(("In erts_poll_init")); erts_sys_break_event = CreateManualEvent(FALSE); - erts_mtx_init(&break_waiter_lock,"break_waiter_lock"); + erts_mtx_init(&break_waiter_lock, "break_waiter_lock", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); break_happened_event = CreateManualEvent(FALSE); erts_atomic32_init_nob(&break_waiter_state, 0); + HARDTRACEF(("Out erts_poll_init")); +} + +void erts_poll_late_init(void) +{ + erts_tid_t thread; erts_thr_create(&thread, &break_waiter, NULL, NULL); ERTS_UNSET_BREAK_REQUESTED; - HARDTRACEF(("Out erts_poll_init")); } /* * Non windows friendly interface, not used when fd's are not continous */ -void erts_poll_get_selected_events(ErtsPollSet ps, +void erts_poll_get_selected_events(ErtsPollSet *ps, ErtsPollEvents ev[], int len) { int i; HARDTRACEF(("In erts_poll_get_selected_events")); for (i = 0; i < len; ++i) - ev[i] = 0; + ev[i] = ERTS_POLL_EV_NONE; HARDTRACEF(("Out erts_poll_get_selected_events")); } diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index 6f28d513c2..0d1a6d4c87 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -40,7 +40,6 @@ WDD_TYPEDEF(int, driver_exit, (ErlDrvPort, int)); WDD_TYPEDEF(int, driver_failure_eof, (ErlDrvPort)); WDD_TYPEDEF(void, erl_drv_busy_msgq_limits, (ErlDrvPort, ErlDrvSizeT *, ErlDrvSizeT *)); WDD_TYPEDEF(int, driver_select, (ErlDrvPort, ErlDrvEvent, int, int)); -WDD_TYPEDEF(int, driver_event, (ErlDrvPort, ErlDrvEvent,ErlDrvEventData)); WDD_TYPEDEF(int, driver_output, (ErlDrvPort, char *, ErlDrvSizeT)); WDD_TYPEDEF(int, driver_output2, (ErlDrvPort, char *, ErlDrvSizeT ,char *, ErlDrvSizeT)); WDD_TYPEDEF(int, driver_output_binary, (ErlDrvPort, char *, ErlDrvSizeT, ErlDrvBinary*, ErlDrvSizeT, ErlDrvSizeT)); @@ -162,7 +161,7 @@ typedef struct { WDD_FTYPE(driver_failure_eof) *driver_failure_eof; WDD_FTYPE(erl_drv_busy_msgq_limits) *erl_drv_busy_msgq_limits; WDD_FTYPE(driver_select) *driver_select; - WDD_FTYPE(driver_event) *driver_event; + void *REMOVED_driver_event; WDD_FTYPE(driver_output) *driver_output; WDD_FTYPE(driver_output2) *driver_output2; WDD_FTYPE(driver_output_binary) *driver_output_binary; @@ -276,7 +275,6 @@ extern TWinDynDriverCallbacks WinDynDriverCallbacks; #define driver_failure_eof (WinDynDriverCallbacks.driver_failure_eof) #define erl_drv_busy_msgq_limits (WinDynDriverCallbacks.erl_drv_busy_msgq_limits) #define driver_select (WinDynDriverCallbacks.driver_select) -#define driver_event (WinDynDriverCallbacks.driver_event) #define driver_output (WinDynDriverCallbacks.driver_output) #define driver_output2 (WinDynDriverCallbacks.driver_output2) #define driver_output_binary (WinDynDriverCallbacks.driver_output_binary) @@ -414,7 +412,7 @@ do { \ ((W).driver_failure_eof) = driver_failure_eof; \ ((W).erl_drv_busy_msgq_limits) = erl_drv_busy_msgq_limits;\ ((W).driver_select) = driver_select; \ -((W).driver_event) = driver_event; \ +((W).REMOVED_driver_event) = NULL; \ ((W).driver_output) = driver_output; \ ((W).driver_output2) = driver_output2; \ ((W).driver_output_binary) = driver_output_binary; \ diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h index 78005aada9..1f53452d17 100644 --- a/erts/emulator/sys/win32/erl_win_sys.h +++ b/erts/emulator/sys/win32/erl_win_sys.h @@ -311,10 +311,8 @@ typedef long ssize_t; #endif /* Threads */ -#ifdef USE_THREADS int init_async(int); int exit_async(void); -#endif #define ERTS_HAVE_TRY_CATCH 1 diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 28019e306c..0598a12351 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -80,11 +80,9 @@ static int application_type(const wchar_t* originalName, wchar_t fullPath[MAX_PA HANDLE erts_service_event; -#ifdef ERTS_SMP -static erts_smp_tsd_key_t win32_errstr_key; -#endif +static erts_tsd_key_t win32_errstr_key; -static erts_smp_atomic_t pipe_creation_counter; +static erts_atomic_t pipe_creation_counter; /* Results from application_type(_w) is one of */ #define APPL_NONE 0 @@ -94,10 +92,8 @@ static erts_smp_atomic_t pipe_creation_counter; static int driver_write(long, HANDLE, byte*, int); static int create_file_thread(struct async_io* aio, int mode); -#ifdef ERTS_SMP static void close_active_handle(DriverData *, HANDLE handle); static DWORD WINAPI threaded_handle_closer(LPVOID param); -#endif static DWORD WINAPI threaded_reader(LPVOID param); static DWORD WINAPI threaded_writer(LPVOID param); static DWORD WINAPI threaded_exiter(LPVOID param); @@ -136,7 +132,7 @@ static OSVERSIONINFO int_os_version; /* Version information for Win32. */ Disabled the use of CancelIoEx as its been seen to cause problem with some drivers. Not sure what to blame; faulty drivers or some form of invalid use. */ -#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) +#if defined(USE_CANCELIOEX) static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); #endif @@ -145,7 +141,7 @@ static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); - call erl_start() to parse arguments and do other init */ -static erts_smp_atomic_t sys_misc_mem_sz; +static erts_atomic_t sys_misc_mem_sz; HMODULE beam_module = NULL; @@ -196,7 +192,7 @@ Uint erts_sys_misc_mem_sz(void) { Uint res = (Uint) erts_check_io_size(); - res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz); + res += (Uint) erts_atomic_read_mb(&sys_misc_mem_sz); return res; } @@ -450,9 +446,7 @@ typedef struct async_io { * the console for Windows NT). */ HANDLE fd; /* Handle for file or pipe. */ -#ifdef ERTS_SMP int async_io_active; /* if true, a close of the file will signal the event in ov */ -#endif OVERLAPPED ov; /* Control structure for overlapped reading. * When overlapped reading is simulated with * a thread, the fields are used as follows: @@ -665,7 +659,7 @@ new_driver_data(ErlDrvPort port_num, int packet_bytes, int wait_objs_required, i dp->inbuf = DRV_BUF_ALLOC(dp->inBufSize); if (dp->inbuf == NULL) goto buf_alloc_error; - erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize); dp->outBufSize = 0; dp->outbuf = NULL; dp->port_num = port_num; @@ -691,7 +685,6 @@ buf_alloc_error: static void release_driver_data(DriverData* dp) { -#ifdef ERTS_SMP #ifdef USE_CANCELIOEX if (fpCancelIoEx != NULL) { if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { @@ -734,18 +727,10 @@ release_driver_data(DriverData* dp) DEBUGF(("...done\n")); } } -#else - if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { - CancelIo(dp->in.fd); - } - if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { - CancelIo(dp->out.fd); - } -#endif if (dp->inbuf != NULL) { - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize); DRV_BUF_FREE(dp->inbuf); dp->inBufSize = 0; dp->inbuf = NULL; @@ -753,8 +738,8 @@ release_driver_data(DriverData* dp) ASSERT(dp->inBufSize == 0); if (dp->outbuf != NULL) { - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -777,7 +762,6 @@ release_driver_data(DriverData* dp) unrefer_driver_data(dp); } -#ifdef ERTS_SMP struct handles_to_be_closed { HANDLE handles[MAXIMUM_WAIT_OBJECTS]; @@ -870,7 +854,6 @@ threaded_handle_closer(LPVOID param) DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc)); return 0; } -#endif /* ERTS_SMP */ /* * Stores input and output file descriptors in the DriverData structure, @@ -946,9 +929,7 @@ init_async_io(DriverData *dp, AsyncIo* aio, int use_threads) aio->flushReplyEvent = NULL; aio->pendingError = 0; aio->bytesTransferred = 0; -#ifdef ERTS_SMP aio->async_io_active = 0; -#endif aio->ov.hEvent = CreateManualEvent(FALSE); if (aio->ov.hEvent == NULL) return -1; @@ -1029,9 +1010,7 @@ async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead) ResetEvent(aio->ov.hEvent); SetEvent(aio->ioAllowed); } else { -#ifdef ERTS_SMP aio->async_io_active = 1; /* Will get 0 when the event actually happened */ -#endif if (ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, &aio->ov)) { DEBUGF(("async_read_file: ReadFile() suceeded: %d bytes\n", @@ -1079,16 +1058,12 @@ async_write_file(AsyncIo* aio, /* Pointer to async control block. */ ResetEvent(aio->ov.hEvent); SetEvent(aio->ioAllowed); } else { -#ifdef ERTS_SMP aio->async_io_active = 1; /* Will get 0 when the event actually happened */ -#endif if (WriteFile(aio->fd, buf, numToWrite, &aio->bytesTransferred, &aio->ov)) { DEBUGF(("async_write_file: WriteFile() suceeded: %d bytes\n", aio->bytesTransferred)); -#ifdef ERTS_SMP aio->async_io_active = 0; /* The event will not be signalled */ -#endif ResetEvent(aio->ov.hEvent); return TRUE; } else { @@ -1190,7 +1165,7 @@ static int spawn_init(void) { int i; -#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) +#if defined(USE_CANCELIOEX) HMODULE module = GetModuleHandle("kernel32"); fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) ((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL); @@ -1762,7 +1737,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o * Otherwise, create named pipes. */ - calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter); + calls = (UWord) erts_atomic_inc_read_nob(&pipe_creation_counter); erts_snprintf(pipe_name, sizeof(pipe_name), "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls); @@ -2447,7 +2422,7 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len) } dp->outBufSize = pb+len; - erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize); /* * Store header bytes (if any). @@ -2476,8 +2451,8 @@ output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len) } else { dp->out.ov.Offset += pb+len; /* For vanilla driver. */ /* XXX OffsetHigh should be changed too. */ - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2511,11 +2486,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) int pb; pb = dp->packet_bytes; -#ifdef ERTS_SMP if(dp->in.thread == (HANDLE) -1) { dp->in.async_io_active = 0; } -#endif DEBUGF(("ready_input: dp %p, event 0x%x\n", dp, ready_event)); /* @@ -2590,8 +2563,8 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) error = ERROR_NOT_ENOUGH_MEMORY; break; /* Break out of loop into error handler. */ } - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, dp->totalNeeded - dp->inBufSize); dp->inBufSize = dp->totalNeeded; dp->inbuf = new_buf; @@ -2680,11 +2653,9 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) DriverData *dp = (DriverData *) drv_data; int error; -#ifdef ERTS_SMP if(dp->out.thread == (HANDLE) -1) { dp->out.async_io_active = 0; } -#endif DEBUGF(("ready_output(%p, 0x%x)\n", drv_data, ready_event)); set_busy_port(dp->port_num, 0); if (!(dp->outbuf)) { @@ -2692,8 +2663,8 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event) write... */ return; } - ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); + ASSERT(erts_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize); + erts_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize); DRV_BUF_FREE(dp->outbuf); dp->outBufSize = 0; dp->outbuf = NULL; @@ -2743,7 +2714,6 @@ sys_init_io(void) max_files = 2*erts_ptab_max(&erts_port); } -#ifdef ERTS_SMP void erts_sys_main_thread(void) { @@ -2756,7 +2726,6 @@ erts_sys_main_thread(void) WaitForSingleObject(dummy, INFINITE); } } -#endif void erts_sys_alloc_init(void) { @@ -2843,7 +2812,7 @@ Preload* sys_preloaded(void) (num_preloaded+1)*sizeof(Preload)); res_name = erts_alloc(ERTS_ALC_T_PRELOADED, (num_preloaded+1)*sizeof(unsigned)); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, + erts_atomic_add_nob(&sys_misc_mem_sz, (num_preloaded+1)*sizeof(Preload) + (num_preloaded+1)*sizeof(unsigned)); for (i = 0; i < num_preloaded; i++) { @@ -2856,7 +2825,7 @@ Preload* sys_preloaded(void) n = GETWORD(data); data += 2; preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1); - erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1); + erts_atomic_add_nob(&sys_misc_mem_sz, n+1); sys_memcpy(preloaded[i].name, data, n); preloaded[i].name[n] = '\0'; data += n; @@ -2938,11 +2907,7 @@ sys_get_key(int fd) char* win32_errorstr(int error) { -#ifdef SMP - LPTSTR lpBufPtr = erts_smp_tsd_get(win32_errstr_key); -#else - static LPTSTR lpBufPtr = NULL; -#endif + LPTSTR lpBufPtr = erts_tsd_get(win32_errstr_key); if (lpBufPtr) { LocalFree(lpBufPtr); } @@ -2956,9 +2921,7 @@ char* win32_errorstr(int error) 0, NULL); SetLastError(error); -#ifdef ERTS_SMP - erts_smp_tsd_set(win32_errstr_key,lpBufPtr); -#endif + erts_tsd_set(win32_errstr_key,lpBufPtr); return lpBufPtr; } @@ -3131,7 +3094,6 @@ check_supported_os_version(void) #endif } -#ifdef USE_THREADS typedef struct { int sched_bind_data; @@ -3176,37 +3138,39 @@ thr_create_prepare_child(void *vtcdp) erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } -#endif /* USE_THREADS */ void erts_sys_pre_init(void) { -#ifdef USE_THREADS erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; -#endif int_os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&int_os_version); check_supported_os_version(); -#ifdef USE_THREADS eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ eid.thread_create_prepare_func = thr_create_prepare; /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup; - erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); #endif + + erts_thr_init(&eid); + #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); + erts_lcnt_post_thr_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif + erts_init_sys_time_sup(); - erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); + erts_atomic_init_nob(&sys_misc_mem_sz, 0); } void noinherit_std_handle(DWORD type) @@ -3226,11 +3190,9 @@ void erl_sys_init(void) noinherit_std_handle(STD_INPUT_HANDLE); noinherit_std_handle(STD_ERROR_HANDLE); -#ifdef ERTS_SMP - erts_smp_tsd_key_create(&win32_errstr_key,"win32_errstr_key"); + erts_tsd_key_create(&win32_errstr_key,"win32_errstr_key"); InitializeCriticalSection(&htbc_lock); -#endif - erts_smp_atomic_init_nob(&pipe_creation_counter,0); + erts_atomic_init_nob(&pipe_creation_counter,0); /* * Test if we have named pipes or not. */ @@ -3273,42 +3235,16 @@ void erl_sys_init(void) SetStdHandle(STD_ERROR_HANDLE, GetStdHandle(STD_OUTPUT_HANDLE)); } erts_sys_init_float(); - erts_init_check_io(); /* Suppress windows error message popups */ SetErrorMode(SetErrorMode(0) | SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); } +void erts_poll_late_init(void); void erl_sys_late_init(void) { /* do nothing */ + erts_poll_late_init(); } - -void -erts_sys_schedule_interrupt(int set) -{ - erts_check_io_interrupt(set); -} - -#ifdef ERTS_SMP -void -erts_sys_schedule_interrupt_timed(int set, ErtsMonotonicTime timeout_time) -{ - erts_check_io_interrupt_timed(set, timeout_time); -} -#endif - -/* - * Called from schedule() when it runs out of runnable processes, - * or when Erlang code has performed INPUT_REDUCTIONS reduction - * steps. runnable == 0 iff there are no runnable Erlang processes. - */ -void -erl_sys_schedule(int runnable) -{ - erts_check_io(!runnable); - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); -} - diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index 21ef71ad9a..5792816267 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -32,22 +32,23 @@ static WCHAR **env_to_arg(WCHAR *env); static WCHAR **find_arg(WCHAR **arg, WCHAR *str);
static int compare(const void *a, const void *b);
-static erts_smp_rwmtx_t environ_rwmtx;
+static erts_rwmtx_t environ_rwmtx; void
erts_sys_env_init(void)
{
- erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+ erts_rwmtx_init(&environ_rwmtx, "environ", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
}
int
erts_sys_putenv_raw(char *key, char *value)
{
int res;
- erts_smp_rwmtx_rwlock(&environ_rwmtx);
+ erts_rwmtx_rwlock(&environ_rwmtx); res = (SetEnvironmentVariable((LPCTSTR) key,
(LPCTSTR) value) ? 0 : 1);
- erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+ erts_rwmtx_rwunlock(&environ_rwmtx); return res;
}
@@ -57,10 +58,10 @@ erts_sys_putenv(char *key, char *value) int res;
WCHAR *wkey = (WCHAR *) key;
WCHAR *wvalue = (WCHAR *) value;
- erts_smp_rwmtx_rwlock(&environ_rwmtx);
+ erts_rwmtx_rwlock(&environ_rwmtx); res = (SetEnvironmentVariableW(wkey,
wvalue) ? 0 : 1);
- erts_smp_rwmtx_rwunlock(&environ_rwmtx);
+ erts_rwmtx_rwunlock(&environ_rwmtx); return res;
}
@@ -75,12 +76,12 @@ erts_sys_getenv(char *key, char *value, size_t *size) DWORD wsize = *size / (sizeof(WCHAR) / sizeof(char));
SetLastError(0);
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); new_size = GetEnvironmentVariableW(wkey,
wvalue,
(DWORD) wsize);
res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); if (res < 0)
return res;
res = new_size > wsize ? 1 : 0;
@@ -110,22 +111,22 @@ int erts_sys_getenv_raw(char *key, char *value, size_t *size)
{
int res;
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); res = erts_sys_getenv__(key, value, size);
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return res;
}
void init_getenv_state(GETENV_STATE *state)
{
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); state->environment_strings = GetEnvironmentStringsW();
state->next_string = state->environment_strings;
}
char *getenv_string(GETENV_STATE *state)
{
- ERTS_SMP_LC_ASSERT(erts_smp_lc_rwmtx_is_rlocked(&environ_rwmtx));
+ ERTS_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&environ_rwmtx)); if (state->next_string[0] == L'\0') {
return NULL;
} else {
@@ -139,7 +140,7 @@ void fini_getenv_state(GETENV_STATE *state) {
FreeEnvironmentStringsW(state->environment_strings);
state->environment_strings = state->next_string = NULL;
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); }
int erts_sys_unsetenv(char *key)
@@ -148,7 +149,7 @@ int erts_sys_unsetenv(char *key) WCHAR *wkey = (WCHAR *) key;
SetLastError(0);
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); GetEnvironmentVariableW(wkey,
NULL,
0);
@@ -156,7 +157,7 @@ int erts_sys_unsetenv(char *key) res = (SetEnvironmentVariableW(wkey,
NULL) ? 0 : 1);
}
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return res;
}
@@ -170,12 +171,12 @@ win_build_environment(char* new_env) tmp_new = (WCHAR *) new_env;
- erts_smp_rwmtx_rlock(&environ_rwmtx);
+ erts_rwmtx_rlock(&environ_rwmtx); tmp = GetEnvironmentStringsW();
merged = merge_environment(tmp, tmp_new);
FreeEnvironmentStringsW(tmp);
- erts_smp_rwmtx_runlock(&environ_rwmtx);
+ erts_rwmtx_runlock(&environ_rwmtx); return (char *) merged;
}
}
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index df838960eb..02aa50500f 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -35,17 +35,11 @@ # define WIN_SYS_INLINE __forceinline #endif -#ifdef ERTS_SMP -erts_smp_atomic32_t erts_break_requested; +erts_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) -#else -volatile int erts_break_requested = 0; -#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) -#define ERTS_UNSET_BREAK_REQUESTED (erts_break_requested = 0) -#endif + erts_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0) extern int nohup; HANDLE erts_sys_break_event = NULL; @@ -57,14 +51,14 @@ void erts_do_break_handling(void) * therefore, make sure that all threads but this one are blocked before * proceeding! */ - erts_smp_thr_progress_block(); + erts_thr_progress_block(); /* call the break handling function, reset the flag */ do_break(); ResetEvent(erts_sys_break_event); ERTS_UNSET_BREAK_REQUESTED; - erts_smp_thr_progress_unblock(); + erts_thr_progress_unblock(); } diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c index e8c67b3928..25c2ad385c 100644 --- a/erts/emulator/sys/win32/sys_time.c +++ b/erts/emulator/sys/win32/sys_time.c @@ -95,7 +95,7 @@ struct sys_time_internal_state_read_mostly__ { }; struct sys_time_internal_state_write_freq__ { - erts_smp_mtx_t mtime_mtx; + erts_mtx_t mtime_mtx; ULONGLONG wrap; ULONGLONG last_tick_count; }; @@ -187,8 +187,6 @@ os_monotonic_time_gtc32(void) { ErtsMonotonicTime mtime; Uint32 ticks = (Uint32) GetTickCount(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT; @@ -205,8 +203,6 @@ os_times_gtc32(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep) ticks = (Uint32) GetTickCount(); GetSystemTime(&st); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - ticks); mtime = ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); mtime <<= ERTS_GET_TICK_COUNT_TIME_UNIT_SHIFT; @@ -265,8 +261,6 @@ sys_hrtime_gtc32(void) { ErtsSysHrTime time; Uint32 ticks = (Uint32) GetTickCount(); - ERTS_CHK_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, - tick_count); time = (ErtsSysHrTime) ERTS_EXTEND_OS_MONOTONIC_TIME(&internal_state.wr.m.os_mtime_xtnd, ticks); time *= (ErtsSysHrTime) (1000 * 1000); @@ -300,8 +294,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp) module = GetModuleHandle(kernel_dll_name); if (!module) { get_tick_count: - erts_smp_mtx_init(&internal_state.w.f.mtime_mtx, - "os_monotonic_time"); + erts_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC); internal_state.w.f.wrap = 0; internal_state.w.f.last_tick_count = 0; diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index fcd7244ae9..b17170c8b8 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -71,8 +71,10 @@ MODULES= \ hash_SUITE \ hibernate_SUITE \ hipe_SUITE \ + iovec_SUITE \ list_bif_SUITE \ lttng_SUITE \ + lcnt_SUITE \ map_SUITE \ match_spec_SUITE \ module_info_SUITE \ diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 3a721095e2..f0871ead7d 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -65,12 +65,7 @@ mseg_clear_cache(Cfg) -> drv_case(Cfg). cpool(Cfg) -> drv_case(Cfg). migration(Cfg) -> - case erlang:system_info(smp_support) of - true -> - drv_case(Cfg, concurrent, "+MZe true"); - false -> - {skipped, "No smp"} - end. + drv_case(Cfg, concurrent, "+MZe true"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl index 6a54fa87e0..bdf8f6c34e 100644 --- a/erts/emulator/test/beam_SUITE.erl +++ b/erts/emulator/test/beam_SUITE.erl @@ -113,20 +113,41 @@ packed_registers(Config) when is_list(Config) -> VarName = list_to_atom("M"++integer_to_list(V)), merl:var(VarName) end || V <- Seq], + MoreNewVars = [begin + VarName = list_to_atom("MM"++integer_to_list(V)), + merl:var(VarName) + end || V <- Seq], + TupleEls = [?Q("id(_@Value@)") || {_,Value} <- S0], S = [?Q("_@Var = id(_@Value@)") || {Var,Value} <- S0], Code = ?Q(["-module('@Mod@').\n" "-export([f/0]).\n" "f() ->\n" + "Tuple = id({_@TupleEls}),\n" + "{_@MoreNewVars} = Tuple,\n" "_@S,\n" "_ = id(0),\n" "L = [_@Vars],\n" "_ = id(1),\n" "[_@NewVars] = L,\n" %Test get_list/3. "_ = id(2),\n" - "id([_@Vars,_@NewVars]).\n" + "id([_@Vars,_@NewVars,_@MoreNewVars]).\n" "id(I) -> I.\n"]), merl:compile_and_load(Code), - CombinedSeq = Seq ++ Seq, + + %% Optionally print the generated code. + PrintCode = false, %Change to true to print code. + + case PrintCode of + false -> + ok; + true -> + merl:print(Code), + erts_debug:df(Mod), + {ok,Dis} = file:read_file(atom_to_list(Mod)++".dis"), + io:put_chars(Dis) + end, + + CombinedSeq = Seq ++ Seq ++ Seq, CombinedSeq = Mod:f(), %% Clean up. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 339c827602..04b7f2de15 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("kernel/include/file.hrl"). -export([all/0, suite/0, - display/1, display_huge/0, + display/1, display_huge/0, display_string/1, erl_bif_types/1,guard_bifs_in_erl_bif_types/1, shadow_comments/1,list_to_utf8_atom/1, specs/1,improper_bif_stubs/1,auto_imports/1, @@ -43,7 +43,7 @@ all() -> [erl_bif_types, guard_bifs_in_erl_bif_types, shadow_comments, specs, improper_bif_stubs, auto_imports, t_list_to_existing_atom, os_env, otp_7526, - display, list_to_utf8_atom, + display, display_string, list_to_utf8_atom, atom_to_binary, binary_to_atom, binary_to_existing_atom, erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, error_stacktrace, error_stacktrace_during_call_trace]. @@ -68,6 +68,28 @@ deeep(N,Acc) -> deeep(N) -> deeep(N,[hello]). +display_string(Config) when is_list(Config) -> + true = erlang:display_string("hej"), + true = erlang:display_string(""), + true = erlang:display_string("hopp"), + true = erlang:display_string("\n"), + true = erlang:display_string(lists:seq(1100,1200)), + {error,badarg} = try + erlang:display_string(atom), + ok + catch + T0:E0 -> + {T0, E0} + end, + {error,badarg} = try + erlang:display_string(make_ref()), + ok + catch + T1:E1 -> + {T1, E1} + end, + ok. + erl_bif_types(Config) when is_list(Config) -> ensure_erl_bif_types_compiled(), @@ -503,6 +525,8 @@ binary_to_atom(Config) when is_list(Config) -> ?BADARG(binary_to_atom(id(<<255>>), utf8)), ?BADARG(binary_to_atom(id(<<255,0>>), utf8)), ?BADARG(binary_to_atom(id(<<16#C0,16#80>>), utf8)), %Overlong 0. + <<B:1/binary, _/binary>> = id(<<194, 163>>), %Truncated character ERL-474 + ?BADARG(binary_to_atom(B, utf8)), %% system_limit failures. ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255>>), utf8)), @@ -691,6 +715,9 @@ erlang_halt(Config) when is_list(Config) -> {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]), {ok,N4} = slave:start(H, halt_node4), {badrpc,nodedown} = rpc:call(N4, erlang, halt, [lists:duplicate(300,$x)]), + %% Test unicode slogan + {ok,N4} = slave:start(H, halt_node4), + {badrpc,nodedown} = rpc:call(N4, erlang, halt, [[339,338,254,230,198,295,167,223,32,12507,12531,12480]]), % This test triggers a segfault when dumping a crash dump % to make sure that we can handle it properly. diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl index 402751393a..5939d024ae 100644 --- a/erts/emulator/test/big_SUITE.erl +++ b/erts/emulator/test/big_SUITE.erl @@ -24,6 +24,7 @@ -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1, borders/1, negative/1, big_float_1/1, big_float_2/1, + bxor_2pow/1, shift_limit_1/1, powmod/1, system_limit/1, toobig/1, otp_6692/1]). %% Internal exports. @@ -42,6 +43,7 @@ suite() -> all() -> [t_div, eq_28, eq_32, eq_big, eq_math, big_literals, borders, negative, {group, big_float}, shift_limit_1, + bxor_2pow, powmod, system_limit, toobig, otp_6692]. groups() -> @@ -337,6 +339,13 @@ system_limit(Config) when is_list(Config) -> {'EXIT',{system_limit,_}} = (catch apply(erlang, id('bsl'), [Maxbig,2])), {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 45)), {'EXIT',{system_limit,_}} = (catch id(1) bsl (1 bsl 69)), + + %% There should be no system_limit exception when shifting a zero. + 0 = id(0) bsl (1 bsl 128), + 0 = id(0) bsr -(1 bsl 128), + Erlang = id(erlang), + 0 = Erlang:'bsl'(id(0), 1 bsl 128), + 0 = Erlang:'bsr'(id(0), -(1 bsl 128)), ok. maxbig() -> @@ -396,3 +405,54 @@ loop2(X,Y,N,M) -> end, loop2(X,Y,N+1,M). + +%% ERL-450 +bxor_2pow(_Config) -> + IL = lists:seq(8*3, 8*16, 4), + JL = lists:seq(0, 64), + [bxor_2pow_1((1 bsl I), (1 bsl J)) + || I <- IL, J <- JL], + ok. + +bxor_2pow_1(A, B) -> + for(-1,1, fun(Ad) -> + for(-1,1, fun(Bd) -> + bxor_2pow_2(A+Ad, B+Bd), + bxor_2pow_2(-A+Ad, B+Bd), + bxor_2pow_2(A+Ad, -B+Bd), + bxor_2pow_2(-A+Ad, -B+Bd) + end) + end). + +for(From, To, _Fun) when From > To -> + ok; +for(From, To, Fun) -> + Fun(From), + for(From+1, To, Fun). + +bxor_2pow_2(A, B) -> + Correct = my_bxor(A, B), + case A bxor B of + Correct -> ok; + Wrong -> + io:format("~.16b bxor ~.16b\n", [A,B]), + io:format("Expected ~.16b\n", [Correct]), + io:format("Got ~.16b\n", [Wrong]), + ct:fail({failed, 'bxor'}) + + end. + +%% Implement bxor without bxor +my_bxor(A, B) -> + my_bxor(A, B, 0, 0). + +my_bxor(0, 0, _, Acc) -> Acc; +my_bxor(-1, -1, _, Acc) -> Acc; +my_bxor(-1, 0, N, Acc) -> (-1 bsl N) bor Acc; % sign extension +my_bxor(0, -1, N, Acc) -> (-1 bsl N) bor Acc; % sign extension +my_bxor(A, B, N, Acc0) -> + Acc1 = case (A band 1) =:= (B band 1) of + true -> Acc0; + false -> Acc0 bor (1 bsl N) + end, + my_bxor(A bsr 1, B bsr 1, N+1, Acc1). diff --git a/erts/emulator/test/big_SUITE_data/borders.dat b/erts/emulator/test/big_SUITE_data/borders.dat index 52e4f35861..c38ff93383 100644 --- a/erts/emulator/test/big_SUITE_data/borders.dat +++ b/erts/emulator/test/big_SUITE_data/borders.dat @@ -1114,3 +1114,38 @@ 1 = 16#800000000000001 rem (-16#800000000000000). 0 = 16#FFFFFFFFFFFFFFF800000000 rem 16#FFFFFFFFFFFFFFF80. +% ERL-450 bxor of big negative 2-pow +-(1 bsl 8) bxor -1 = 16#ff. +-(1 bsl 16) bxor -1 = 16#ffff. +-(1 bsl 24) bxor -1 = 16#ffffff. +-(1 bsl 32) bxor -1 = 16#ffffffff. +-(1 bsl 40) bxor -1 = 16#ffffffffff. +-(1 bsl 48) bxor -1 = 16#ffffffffffff. +-(1 bsl 56) bxor -1 = 16#ffffffffffffff. +-(1 bsl 64) bxor -1 = 16#ffffffffffffffff. +-(1 bsl 72) bxor -1 = 16#ffffffffffffffffff. +-(1 bsl 80) bxor -1 = 16#ffffffffffffffffffff. +-(1 bsl 88) bxor -1 = 16#ffffffffffffffffffffff. +-(1 bsl 96) bxor -1 = 16#ffffffffffffffffffffffff. +-(1 bsl 104) bxor -1 = 16#ffffffffffffffffffffffffff. +-(1 bsl 112) bxor -1 = 16#ffffffffffffffffffffffffffff. +-(1 bsl 120) bxor -1 = 16#ffffffffffffffffffffffffffffff. +-(1 bsl 128) bxor -1 = 16#ffffffffffffffffffffffffffffffff. +-(1 bsl 136) bxor -1 = 16#ffffffffffffffffffffffffffffffffff. +-(1 bsl 8) bxor 1 = -16#ff. +-(1 bsl 16) bxor 1 = -16#ffff. +-(1 bsl 24) bxor 1 = -16#ffffff. +-(1 bsl 32) bxor 1 = -16#ffffffff. +-(1 bsl 40) bxor 1 = -16#ffffffffff. +-(1 bsl 48) bxor 1 = -16#ffffffffffff. +-(1 bsl 56) bxor 1 = -16#ffffffffffffff. +-(1 bsl 64) bxor 1 = -16#ffffffffffffffff. +-(1 bsl 72) bxor 1 = -16#ffffffffffffffffff. +-(1 bsl 80) bxor 1 = -16#ffffffffffffffffffff. +-(1 bsl 88) bxor 1 = -16#ffffffffffffffffffffff. +-(1 bsl 96) bxor 1 = -16#ffffffffffffffffffffffff. +-(1 bsl 104) bxor 1 = -16#ffffffffffffffffffffffffff. +-(1 bsl 112) bxor 1 = -16#ffffffffffffffffffffffffffff. +-(1 bsl 120) bxor 1 = -16#ffffffffffffffffffffffffffffff. +-(1 bsl 128) bxor 1 = -16#ffffffffffffffffffffffffffffffff. +-(1 bsl 136) bxor 1 = -16#ffffffffffffffffffffffffffffffffff. diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 4d17276e5c..61536bacd7 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -599,6 +599,9 @@ bad_binary_to_term(Config) when is_list(Config) -> %% Bad float. bad_bin_to_term(<<131,70,-1:64>>), + + %% Truncated UTF8 character (ERL-474) + bad_bin_to_term(<<131,119,1,194,163>>), ok. bad_bin_to_term(BadBin) -> diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index b79f4b995d..ce50bcdd86 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -905,14 +905,28 @@ bs_add_overflow(_Config) -> _ when Memsize < (2 bsl 30) -> {skip, "Less then 2 GB of memory"}; 4 -> - Large = <<0:((1 bsl 30)-1)>>, - {'EXIT',{system_limit,_}} = - (catch <<Large/bits, Large/bits, Large/bits, Large/bits, - Large/bits, Large/bits, Large/bits, Large/bits, - Large/bits>>), + {'EXIT', {system_limit, _}} = (catch bs_add_overflow_signed()), + {'EXIT', {system_limit, _}} = (catch bs_add_overflow_unsigned()), ok end. +bs_add_overflow_signed() -> + %% Produce a large result of bs_add that, if cast to signed int, would + %% overflow into a negative number that fits a smallnum. + Large = <<0:((1 bsl 30)-1)>>, + <<Large/bits, Large/bits, Large/bits, Large/bits, + Large/bits, Large/bits, Large/bits, Large/bits, + Large/bits>>. + +bs_add_overflow_unsigned() -> + %% Produce a large result of bs_add that goes beyond the limit of an + %% unsigned word. This used to succeed but produced an incorrect result + %% where B =:= C! + A = <<0:((1 bsl 32)-8)>>, + B = <<2, 3>>, + C = <<A/binary,1,B/binary>>, + true = byte_size(B) < byte_size(C). + id(I) -> I. memsize() -> diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index b4ec99f902..17cd1d1a3b 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -56,6 +56,7 @@ bad_dist_ext_process_info/1, bad_dist_ext_control/1, bad_dist_ext_connection_id/1, + bad_dist_ext_size/1, start_epmd_false/1, epmd_module/1]). %% Internal exports. @@ -92,6 +93,7 @@ groups() -> [dist_auto_connect_never, dist_auto_connect_once]}, {bad_dist_ext, [], [bad_dist_ext_receive, bad_dist_ext_process_info, + bad_dist_ext_size, bad_dist_ext_control, bad_dist_ext_connection_id]}]. %% Tests pinging a node in different ways. @@ -418,18 +420,20 @@ make_busy(Node, Time) when is_integer(Time) -> Own = 500, freeze_node(Node, Time+Own), Data = make_busy_data(), + DCtrl = dctrl(Node), %% first make port busy Pid = spawn_link(fun () -> forever(fun () -> - dport_reg_send(Node, - '__noone__', - Data) + dctrl_dop_reg_send(Node, + '__noone__', + Data) end) end), receive after Own -> ok end, until(fun () -> - case process_info(Pid, status) of - {status, suspended} -> true; + case {DCtrl, process_info(Pid, status)} of + {DPrt, {status, suspended}} when is_port(DPrt) -> true; + {DPid, {status, waiting}} when is_pid(DPid) -> true; _ -> false end end), @@ -1672,6 +1676,57 @@ bad_dist_ext_connection_id(Config) when is_list(Config) -> stop_node(Offender), stop_node(Victim). +%% OTP-14661: Bad message is discovered by erts_msg_attached_data_size +bad_dist_ext_size(Config) when is_list(Config) -> + {ok, Offender} = start_node(bad_dist_ext_process_info_offender), + %%Prog = "Prog=/home/uabseri/src/otp_new3/bin/cerl -rr -debug", + Prog = [], + {ok, Victim} = start_node(bad_dist_ext_process_info_victim, [], Prog), + start_node_monitors([Offender,Victim]), + + Parent = self(), + P = spawn_link(Victim, + fun () -> + Parent ! {self(), started}, + receive check_msgs -> ok end, %% DID CRASH HERE + bad_dist_ext_check_msgs([one]), + Parent ! {self(), messages_checked} + end), + + receive {P, started} -> ok end, + P ! one, + + Suspended = make_ref(), + S = spawn(Victim, + fun () -> + erlang:suspend_process(P), + Parent ! Suspended, + receive after infinity -> ok end + end), + + receive Suspended -> ok end, + pong = rpc:call(Victim, net_adm, ping, [Offender]), + verify_up(Offender, Victim), + send_bad_msgs(Offender, P, 1, dmsg_bad_tag()), + + %% Make sure bad msgs has reached Victim + rpc:call(Offender, rpc, call, [Victim, erlang, node, []]), + + verify_still_up(Offender, Victim), + + rpc:call(Victim, erlang, process_info, [P, total_heap_size]), + + verify_down(Offender, connection_closed, Victim, killed), + + P ! check_msgs, + exit(S, bang), % resume Victim + receive {P, messages_checked} -> ok end, + + unlink(P), + verify_no_down(Offender, Victim), + stop_node(Offender), + stop_node(Victim). + bad_dist_struct_check_msgs([]) -> receive @@ -1703,37 +1758,38 @@ bad_dist_ext_check_msgs([M|Ms]) -> bad_dist_ext_check_msgs(Ms) end. +ensure_dctrl(Node) -> + case dctrl(Node) of + undefined -> + pong = net_adm:ping(Node), + dctrl(Node); + DCtrl -> + DCtrl + end. -dport_reg_send(Node, Name, Msg) -> - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_REG_SEND, - self(), - ?COOKIE, - Name}), - dmsg_ext(Msg)]). - - -dport_send(To, Msg) -> +dctrl_send(DPrt, Data) when is_port(DPrt) -> + port_command(DPrt, Data); +dctrl_send(DPid, Data) when is_pid(DPid) -> + Ref = make_ref(), + DPid ! {send, self(), Ref, Data}, + receive {Ref, Res} -> Res end. + +dctrl_dop_reg_send(Node, Name, Msg) -> + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_REG_SEND, + self(), + ?COOKIE, + Name}), + dmsg_ext(Msg)]). + +dctrl_dop_send(To, Msg) -> Node = node(To), - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_SEND, - ?COOKIE, - To}), - dmsg_ext(Msg)]). + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_SEND, ?COOKIE, To}), + dmsg_ext(Msg)]). + send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) -> send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]). send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> @@ -1743,7 +1799,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> fun () -> Node = node(Victim), pong = net_adm:ping(Node), - DPrt = dport(Node), + DCtrl = dctrl(Node), Bad1 = case WhereToPutSelf of 0 -> Bad; @@ -1756,7 +1812,7 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> [] -> []; _Other -> [dmsg_ext(PayLoad)] end, - port_command(DPrt, DData), + dctrl_send(DCtrl, DData), Parent ! {DData,Done} end), receive @@ -1775,20 +1831,23 @@ send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> send_bad_msg(BadNode, To) -> send_bad_msgs(BadNode, To, 1). -send_bad_msgs(BadNode, To, Repeat) when is_atom(BadNode), - is_pid(To), - is_integer(Repeat) -> +send_bad_msgs(BadNode, To, Repeat) -> + send_bad_msgs(BadNode, To, Repeat, dmsg_bad_atom_cache_ref()). + +send_bad_msgs(BadNode, To, Repeat, BadTerm) when is_atom(BadNode), + is_pid(To), + is_integer(Repeat) -> Parent = self(), Done = make_ref(), spawn_link(BadNode, fun () -> Node = node(To), pong = net_adm:ping(Node), - DPrt = dport(Node), + DCtrl = dctrl(Node), DData = [dmsg_hdr(), dmsg_ext({?DOP_SEND, ?COOKIE, To}), - dmsg_bad_atom_cache_ref()], - repeat(fun () -> port_command(DPrt, DData) end, Repeat), + BadTerm], + repeat(fun () -> dctrl_send(DCtrl, DData) end, Repeat), Parent ! Done end), receive Done -> ok end. @@ -1810,11 +1869,12 @@ send_bad_ctl(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) -> replace}), CtlBeginSize = size(Ctl) - size(Replace), <<CtlBegin:CtlBeginSize/binary, Replace/binary>> = Ctl, - port_command(dport(ToNode), - [dmsg_fake_hdr2(), - CtlBegin, - dmsg_bad_atom_cache_ref(), - dmsg_ext({a, message})]), + DCtrl = dctrl(ToNode), + Data = [dmsg_fake_hdr2(), + CtlBegin, + dmsg_bad_atom_cache_ref(), + dmsg_ext({a, message})], + dctrl_send(DCtrl, Data), Parent ! Done end), receive Done -> ok end. @@ -1827,17 +1887,17 @@ send_bad_dhdr(BadNode, ToNode) when is_atom(BadNode), is_atom(ToNode) -> spawn_link(BadNode, fun () -> pong = net_adm:ping(ToNode), - port_command(dport(ToNode), dmsg_bad_hdr()), + dctrl_send(dctrl(ToNode), dmsg_bad_hdr()), Parent ! Done end), receive Done -> ok end. -dport(Node) when is_atom(Node) -> +dctrl(Node) when is_atom(Node) -> case catch erts_debug:get_internal_state(available_internal_state) of true -> true; _ -> erts_debug:set_internal_state(available_internal_state, true) end, - erts_debug:get_internal_state({dist_port, Node}). + erts_debug:get_internal_state({dist_ctrl, Node}). dmsg_hdr() -> [131, % Version Magic @@ -1874,6 +1934,9 @@ dmsg_ext(Term) -> dmsg_bad_atom_cache_ref() -> [$R, 137]. +dmsg_bad_tag() -> %% Will fail early at heap size calculation + [$?, 66]. + start_epmd_false(Config) when is_list(Config) -> %% Start a node with the option -start_epmd false. {ok, OtherNode} = start_node(start_epmd_false, "-start_epmd false"), @@ -1979,7 +2042,7 @@ freeze_node(Node, MS) -> fun () -> erts_debug:set_internal_state(available_internal_state, true), - dport_send(Freezer, DoingIt), + dctrl_dop_send(Freezer, DoingIt), receive after Own -> ok end, erts_debug:set_internal_state(block, MS+Own) end), diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 6810729285..33d0b708cf 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -43,9 +43,9 @@ outputv_errors/1, driver_unloaded/1, io_ready_exit/1, + use_fallback_pollset/0, use_fallback_pollset/1, bad_fd_in_pollset/1, - driver_event/1, fd_change/1, steal_control/1, otp_6602/1, @@ -58,11 +58,9 @@ ioq_exit_ready_output/1, ioq_exit_timeout/1, ioq_exit_ready_async/1, - ioq_exit_event/1, ioq_exit_ready_input_async/1, ioq_exit_ready_output_async/1, ioq_exit_timeout_async/1, - ioq_exit_event_async/1, zero_extended_marker_garb_drv/1, invalid_extended_marker_drv/1, larger_major_vsn_drv/1, @@ -86,6 +84,8 @@ -export([bin_prefix/2]). +-export([get_check_io_total/1]). % for z_SUITE.erl + -include_lib("common_test/include/ct.hrl"). @@ -119,17 +119,26 @@ -define(heap_binary_size, 64). init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> - case catch erts_debug:get_internal_state(available_internal_state) of - true -> ok; - _ -> erts_debug:set_internal_state(available_internal_state, true) - end, + CIOD = rpc(Config, + fun() -> + case catch erts_debug:get_internal_state(available_internal_state) of + true -> ok; + _ -> erts_debug:set_internal_state(available_internal_state, true) + end, + erts_debug:get_internal_state(check_io_debug) + end), erlang:display({init_per_testcase, Case}), - 0 = element(1, erts_debug:get_internal_state(check_io_debug)), + 0 = element(1, CIOD), [{testcase, Case}|Config]. -end_per_testcase(Case, _Config) -> +end_per_testcase(Case, Config) -> erlang:display({end_per_testcase, Case}), - 0 = element(1, erts_debug:get_internal_state(check_io_debug)), + CIOD = rpc(Config, + fun() -> + get_stable_check_io_info(), + erts_debug:get_internal_state(check_io_debug) + end), + 0 = element(1, CIOD), ok. suite() -> @@ -137,10 +146,13 @@ suite() -> {timetrap, {minutes, 1}}]. all() -> %% Keep a_test first and z_test last... - [a_test, outputv_errors, outputv_echo, queue_echo, {group, timer}, - driver_unloaded, io_ready_exit, use_fallback_pollset, - bad_fd_in_pollset, driver_event, fd_change, - steal_control, otp_6602, driver_system_info_base_ver, + [a_test, outputv_errors, outputv_echo, queue_echo, + {group, timer}, + driver_unloaded, io_ready_exit, otp_6602, + {group, polling}, + {group, poll_thread}, + {group, poll_set}, + driver_system_info_base_ver, driver_system_info_prev_ver, driver_system_info_current_ver, driver_monitor, {group, ioq_exit}, zero_extended_marker_garb_drv, @@ -148,7 +160,6 @@ all() -> %% Keep a_test first and z_test last... larger_minor_vsn_drv, smaller_major_vsn_drv, smaller_minor_vsn_drv, peek_non_existing_queue, otp_6879, caller, many_events, missing_callbacks, - smp_select, driver_select_use, thread_mseg_alloc_cache_clean, otp_9302, thr_free_drv, @@ -161,11 +172,18 @@ groups() -> [{timer, [], [timer_measure, timer_cancel, timer_delay, timer_change]}, + {poll_thread, [], [{group, polling}]}, + {poll_set, [], [{group, polling}]}, + {polling, [], + [a_test, use_fallback_pollset, + bad_fd_in_pollset, fd_change, + steal_control, smp_select, + driver_select_use, z_test]}, {ioq_exit, [], [ioq_exit_ready_input, ioq_exit_ready_output, - ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event, + ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_ready_input_async, ioq_exit_ready_output_async, - ioq_exit_timeout_async, ioq_exit_event_async]}]. + ioq_exit_timeout_async]}]. init_per_suite(Config) -> Config. @@ -173,10 +191,28 @@ init_per_suite(Config) -> end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false). +init_per_group(poll_thread, Config) -> + [{node_args, "+IOt 2"} | Config]; +init_per_group(poll_set, Config) -> + [{node_args, "+IOt 2 +IOp 2"} | Config]; +init_per_group(polling, Config) -> + case proplists:get_value(node_args, Config) of + undefined -> + Config; + Args -> + {ok, Node} = start_node(polling, Args), + [{node, Node} | Config] + end; init_per_group(_GroupName, Config) -> Config. end_per_group(_GroupName, Config) -> + case proplists:get_value(node, Config) of + undefined -> + ok; + Node -> + stop_node(Node) + end, Config. %% Test sending bad types to port with an outputv-capable driver. @@ -778,21 +814,23 @@ io_ready_exit(Config) when is_list(Config) -> -define(CHKIO_STOP, 0). -define(CHKIO_USE_FALLBACK_POLLSET, 1). -define(CHKIO_BAD_FD_IN_POLLSET, 2). --define(CHKIO_DRIVER_EVENT, 3). -define(CHKIO_FD_CHANGE, 4). -define(CHKIO_STEAL, 5). -define(CHKIO_STEAL_AUX, 6). -define(CHKIO_SMP_SELECT, 7). -define(CHKIO_DRV_USE, 8). +use_fallback_pollset() -> + [{timetrap, {minutes, 2}}]. + use_fallback_pollset(Config) when is_list(Config) -> + rpc(Config, fun() -> use_fallback_pollset_t(Config) end). + +use_fallback_pollset_t(Config) when is_list(Config) -> FlbkFun = fun () -> - ChkIoDuring = erlang:system_info(check_io), - case lists:keysearch(fallback_poll_set_size, - 1, - ChkIoDuring) of - {value, - {fallback_poll_set_size, N}} when N > 0 -> + {Flbk, _} = get_fallback(erlang:system_info(check_io)), + case lists:keysearch(total_poll_set_size, 1, Flbk) of + {value, {total_poll_set_size, N}} when N > 0 -> ok; Error -> ct:fail({failed_to_use_fallback, Error}) @@ -814,6 +852,7 @@ use_fallback_pollset(Config) when is_list(Config) -> Skip -> {fun () -> ok end, Skip, ok} end, + io:format("Node = ~p~n",[node()]), case chkio_test_fini(chkio_test(Handel, ?CHKIO_USE_FALLBACK_POLLSET, fun () -> @@ -825,27 +864,31 @@ use_fallback_pollset(Config) when is_list(Config) -> end. bad_fd_in_pollset(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_BAD_FD_IN_POLLSET, - fun () -> sleep(1000) end)). - -driver_event(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_DRIVER_EVENT, - fun () -> sleep(1000) end)). + rpc(Config, + fun() -> + chkio_test_fini(chkio_test(chkio_test_init(Config), + ?CHKIO_BAD_FD_IN_POLLSET, + fun () -> sleep(1000) end)) + end). fd_change(Config) when is_list(Config) -> - chkio_test_fini(chkio_test(chkio_test_init(Config), - ?CHKIO_FD_CHANGE, - fun () -> sleep(1000) end)). + rpc(Config, + fun() -> + chkio_test_fini(chkio_test(chkio_test_init(Config), + ?CHKIO_FD_CHANGE, + fun () -> sleep(1000) end)) + end). steal_control(Config) when is_list(Config) -> - chkio_test_fini(case chkio_test_init(Config) of - {erts_poll_info, _} = Hndl -> - steal_control_test(Hndl); - Skip -> - Skip - end). + rpc(Config, + fun() -> + chkio_test_fini(case chkio_test_init(Config) of + {erts_poll_info, _} = Hndl -> + steal_control_test(Hndl); + Skip -> + Skip + end) + end). steal_control_test(Hndl = {erts_poll_info, Before}) -> Port = open_chkio_port(), @@ -887,7 +930,7 @@ chkio_test_init(Config) when is_list(Config) -> ChkIo = get_stable_check_io_info(), case catch lists:keysearch(name, 1, ChkIo) of {value, {name, erts_poll}} -> - io:format("Before test: ~p~n", [ChkIo]), + ct:log("Before test: ~p~n", [ChkIo]), Path = proplists:get_value(data_dir, Config), erl_ddll:start(), ok = load_driver(Path, 'chkio_drv'), @@ -948,8 +991,9 @@ chkio_test({erts_poll_info, Before}, "ok" -> chk_chkio_port(Port), Fun(), - During = erlang:system_info(check_io), + During = get_check_io_total(erlang:system_info(check_io)), erlang:display(During), + 0 = element(1, erts_debug:get_internal_state(check_io_debug)), io:format("During test: ~p~n", [During]), chk_chkio_port(Port), @@ -1001,22 +1045,83 @@ verify_chkio_state(Before, After) -> ok. get_stable_check_io_info() -> - ChkIo = erlang:system_info(check_io), - PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of - {value, {pending_updates, PendNo}} -> - PendNo; - false -> - 0 - end, - {value, {active_fds, ActFds}} = lists:keysearch(active_fds, 1, ChkIo), + get_stable_check_io_info(10). +get_stable_check_io_info(0) -> + get_check_io_total(erlang:system_info(check_io)); +get_stable_check_io_info(N) -> + ChkIo = get_check_io_total(erlang:system_info(check_io)), + PendUpdNo = proplists:get_value(pending_updates, ChkIo, 0), + ActFds = proplists:get_value(active_fds, ChkIo), case {PendUpdNo, ActFds} of {0, 0} -> ChkIo; _ -> - receive after 10 -> ok end, - get_stable_check_io_info() + receive after 100 -> ok end, + get_stable_check_io_info(N-1) end. +%% Merge return from erlang:system_info(check_io) +%% as if it was one big pollset. +get_check_io_total(ChkIo) -> + ct:log("ChkIo = ~p~n",[ChkIo]), + {Fallback, Rest} = get_fallback(ChkIo), + add_fallback_infos(Fallback, + lists:foldl(fun(Pollset, Acc) -> + lists:zipwith(fun(A, B) -> + add_pollset_infos(A,B) + end, + Pollset, Acc) + end, + hd(Rest), tl(Rest))). + +add_pollset_infos({Tag, A}=TA , {Tag, B}=TB) -> + case tag_type(Tag) of + sum -> + {Tag, A + B}; + const -> + case A of + B -> TA; + _ -> + ct:fail("Unexpected diff in pollsets ~p != ~p", + [TA,TB]) + end + end. + +get_fallback([MaybeFallback | ChkIo] = AllChkIo) -> + case proplists:get_value(fallback, MaybeFallback) of + true -> + {MaybeFallback, ChkIo}; + false -> + {undefined, AllChkIo} + end. + +add_fallback_infos(undefined, Acc) -> + Acc; +add_fallback_infos(Flbk, Acc) -> + lists:zipwith(fun({Tag, A}=TA, {Tag, B}=TB) -> + case tag_type(Tag) of + sum -> {Tag, A + B}; + const when Tag =:= fallback -> TA; + const -> TB + end + end, + Flbk, Acc). + +tag_type(name) -> const; +tag_type(primary) -> const; +tag_type(fallback) -> const; +tag_type(kernel_poll) -> const; +tag_type(memory_size) -> sum; +tag_type(total_poll_set_size) -> sum; +tag_type(lazy_updates) -> const; +tag_type(pending_updates) -> sum; +tag_type(batch_updates) -> const; +tag_type(concurrent_updates) -> const; +tag_type(max_fds) -> const; +tag_type(active_fds) -> sum; +tag_type(poll_threads) -> sum. + + %% Missed port lock when stealing control of fd from a %% driver that didn't use the same lock. The lock checker %% used to trigger on this and dump core. @@ -1144,8 +1249,6 @@ check_si_res(["thread", "false"]) -> false = erlang:system_info(threads); check_si_res(["smp", "true"]) -> true = erlang:system_info(smp_support); -check_si_res(["smp", "false"]) -> - false = erlang:system_info(smp_support); %% Data added in second version of driver_system_info() (driver version 1.1) check_si_res(["async_thrs", Value]) -> @@ -1338,11 +1441,9 @@ driver_monitor(Config) when is_list(Config) -> -define(IOQ_EXIT_READY_OUTPUT, 2). -define(IOQ_EXIT_TIMEOUT, 3). -define(IOQ_EXIT_READY_ASYNC, 4). --define(IOQ_EXIT_EVENT, 5). -define(IOQ_EXIT_READY_INPUT_ASYNC, 6). -define(IOQ_EXIT_READY_OUTPUT_ASYNC, 7). -define(IOQ_EXIT_TIMEOUT_ASYNC, 8). --define(IOQ_EXIT_EVENT_ASYNC, 9). ioq_exit_test(Config, TestNo) -> Drv = ioq_exit_drv, @@ -1395,9 +1496,6 @@ ioq_exit_timeout(Config) when is_list(Config) -> ioq_exit_ready_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_READY_ASYNC). -ioq_exit_event(Config) when is_list(Config) -> - ioq_exit_test(Config, ?IOQ_EXIT_EVENT). - ioq_exit_ready_input_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_READY_INPUT_ASYNC). @@ -1407,9 +1505,6 @@ ioq_exit_ready_output_async(Config) when is_list(Config) -> ioq_exit_timeout_async(Config) when is_list(Config) -> ioq_exit_test(Config, ?IOQ_EXIT_TIMEOUT_ASYNC). -ioq_exit_event_async(Config) when is_list(Config) -> - ioq_exit_test(Config, ?IOQ_EXIT_EVENT_ASYNC). - vsn_mismatch_test(Config, LoadResult) -> Path = proplists:get_value(data_dir, Config), @@ -1643,7 +1738,7 @@ missing_callbacks(Config) when is_list(Config) -> smp_select(Config) when is_list(Config) -> case os:type() of {win32,_} -> {skipped, "Test not implemented for this OS"}; - _ -> smp_select0(Config) + _ -> rpc(Config, fun() -> smp_select0(Config) end) end. smp_select0(Config) -> @@ -1699,7 +1794,7 @@ smp_select_wait(Pids, TimeoutMsg) -> driver_select_use(Config) when is_list(Config) -> case os:type() of {win32,_} -> {skipped, "Test not implemented for this OS"}; - _ -> driver_select_use0(Config) + _ -> rpc(Config, fun() -> driver_select_use0(Config) end) end. driver_select_use0(Config) -> @@ -1944,44 +2039,39 @@ thr_msg_blast_receiver_proc(Port, Max, Parent, Done) -> end. thr_msg_blast(Config) when is_list(Config) -> - case erlang:system_info(smp_support) of - false -> - {skipped, "Non-SMP emulator; nothing to test..."}; - true -> - Path = proplists:get_value(data_dir, Config), - erl_ddll:start(), - ok = load_driver(Path, thr_msg_blast_drv), - MemBefore = driver_alloc_size(), - Start = os:timestamp(), - Port = open_port({spawn, thr_msg_blast_drv}, []), - true = is_port(Port), - Done = make_ref(), - Me = self(), - spawn(fun () -> - thr_msg_blast_receiver_proc(Port, 1, Me, Done) - end), - receive - Done -> ok - end, - ok = thr_msg_blast_receiver(Port, 0, 32*10000), - port_close(Port), - End = os:timestamp(), - receive - Garbage -> - ct:fail({received_garbage, Port, Garbage}) - after 2000 -> - ok - end, - MemAfter = driver_alloc_size(), - io:format("MemBefore=~p, MemAfter=~p~n", - [MemBefore, MemAfter]), - ThrMsgBlastTime = timer:now_diff(End,Start)/1000000, - io:format("ThrMsgBlastTime=~p~n", [ThrMsgBlastTime]), - MemBefore = MemAfter, - Res = {thr_msg_blast_time, ThrMsgBlastTime}, - erlang:display(Res), - Res - end. + Path = proplists:get_value(data_dir, Config), + erl_ddll:start(), + ok = load_driver(Path, thr_msg_blast_drv), + MemBefore = driver_alloc_size(), + Start = os:timestamp(), + Port = open_port({spawn, thr_msg_blast_drv}, []), + true = is_port(Port), + Done = make_ref(), + Me = self(), + spawn(fun () -> + thr_msg_blast_receiver_proc(Port, 1, Me, Done) + end), + receive + Done -> ok + end, + ok = thr_msg_blast_receiver(Port, 0, 32*10000), + port_close(Port), + End = os:timestamp(), + receive + Garbage -> + ct:fail({received_garbage, Port, Garbage}) + after 2000 -> + ok + end, + MemAfter = driver_alloc_size(), + io:format("MemBefore=~p, MemAfter=~p~n", + [MemBefore, MemAfter]), + ThrMsgBlastTime = timer:now_diff(End,Start)/1000000, + io:format("ThrMsgBlastTime=~p~n", [ThrMsgBlastTime]), + MemBefore = MemAfter, + Res = {thr_msg_blast_time, ThrMsgBlastTime}, + erlang:display(Res), + Res. -define(IN_RANGE(LoW_, VaLuE_, HiGh_), case in_range(LoW_, VaLuE_, HiGh_) of @@ -2271,10 +2361,10 @@ count_proc_sched(Ps, PNs) -> end. a_test(Config) when is_list(Config) -> - check_io_debug(). + rpc(Config, fun check_io_debug/0). z_test(Config) when is_list(Config) -> - check_io_debug(). + rpc(Config, fun check_io_debug/0). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Utilities @@ -2282,8 +2372,8 @@ z_test(Config) when is_list(Config) -> check_io_debug() -> get_stable_check_io_info(), - {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoDrvEvStructs} = CheckIoDebug - = erts_debug:get_internal_state(check_io_debug), + {NoErrorFds, NoUsedFds, NoDrvSelStructs, NoEnifSelStructs} + = CheckIoDebug = erts_debug:get_internal_state(check_io_debug), HasGetHost = has_gethost(), ct:log("check_io_debug: ~p~n" "HasGetHost: ~p",[CheckIoDebug, HasGetHost]), @@ -2296,7 +2386,7 @@ check_io_debug() -> %% one extra used fd that is not selected on ok end, - 0 = NoDrvEvStructs, + 0 = NoEnifSelStructs, ok. has_gethost() -> @@ -2466,15 +2556,19 @@ sleep(Ms) when is_integer(Ms), Ms >= 0 -> start_node(Config) when is_list(Config) -> + start_node(proplists:get_value(testcase, Config)); +start_node(Name) -> + start_node(Name, ""). +start_node(NodeName, Args) -> Pa = filename:dirname(code:which(?MODULE)), Name = list_to_atom(atom_to_list(?MODULE) ++ "-" - ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ atom_to_list(NodeName) ++ "-" ++ integer_to_list(erlang:system_time(second)) ++ "-" ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, slave, [{args, "-pa "++Pa}]). + test_server:start_node(Name, slave, [{args, Args ++ " -pa "++Pa}]). stop_node(Node) -> test_server:stop_node(Node). @@ -2488,14 +2582,6 @@ wait_deallocations() -> end. driver_alloc_size() -> - case erlang:system_info(smp_support) of - true -> - ok; - false -> - %% driver_alloc also used by elements in lock-free queues, - %% give these some time to be deallocated... - receive after 100 -> ok end - end, wait_deallocations(), case erlang:system_info({allocator_sizes, driver_alloc}) of false -> @@ -2515,3 +2601,35 @@ driver_alloc_size() -> Sz0+Sz end, 0, CS) end. + +rpc(Config, Fun) -> + case proplists:get_value(node, Config) of + undefined -> + Fun(); + Node -> + Self = self(), + Ref = make_ref(), + Pid = spawn(Node, + fun() -> + Result + = try Fun() of + Res -> Res + catch E:R -> + {'EXIT',E,R,erlang:get_stacktrace()} + end, + Self ! {Ref, Result} + end), + MRef = monitor(process, Pid), + receive + {'DOWN', MRef, _Type, _Object, Info} -> + erlang:error({died, Pid, Info}); + {Ref, {'EXIT',E,R,ST}} -> + erlang:demonitor(MRef, [flush]), + erlang:raise(E,R,ST); + {Ref, Ret} -> + erlang:demonitor(MRef, [flush]), + Ret; + Other -> + ct:fail(Other) + end + end. diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index 8e5e81665c..d548c4b1dc 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -42,7 +42,6 @@ #define CHKIO_STOP 0 #define CHKIO_USE_FALLBACK_POLLSET 1 #define CHKIO_BAD_FD_IN_POLLSET 2 -#define CHKIO_DRIVER_EVENT 3 #define CHKIO_FD_CHANGE 4 #define CHKIO_STEAL 5 #define CHKIO_STEAL_AUX 6 @@ -67,15 +66,6 @@ typedef struct { } ChkioFallbackData; typedef struct { - int in_fd; - struct erl_drv_event_data in_data; - int in_ok; - int out_fd; - struct erl_drv_event_data out_data; - int out_ok; -} ChkioDriverEvent; - -typedef struct { int fds[2]; int same_fd; } ChkioFdChange; @@ -86,14 +76,10 @@ typedef struct { typedef struct { int driver_select_fds[2]; - int driver_event_fds[2]; - struct erl_drv_event_data event_data[2]; } ChkioSteal; typedef struct { int driver_select_fds[2]; - int driver_event_fds[2]; - struct erl_drv_event_data event_data[2]; } ChkioStealAux; @@ -141,7 +127,6 @@ static ErlDrvData chkio_drv_start(ErlDrvPort, char *); static void chkio_drv_stop(ErlDrvData); static void chkio_drv_ready_input(ErlDrvData, ErlDrvEvent); static void chkio_drv_ready_output(ErlDrvData, ErlDrvEvent); -static void chkio_drv_ready_event(ErlDrvData, ErlDrvEvent, ErlDrvEventData); static ErlDrvSSizeT chkio_drv_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT, char **, ErlDrvSizeT); static void chkio_drv_timeout(ErlDrvData); @@ -164,7 +149,7 @@ static ErlDrvEntry chkio_drv_entry = { NULL, /* ready_async */ NULL, /* flush */ NULL, /* call */ - chkio_drv_ready_event, + NULL, /* unused_event_callback */ ERL_DRV_EXTENDED_MARKER, ERL_DRV_EXTENDED_MAJOR_VERSION, @@ -243,25 +228,6 @@ stop_use_fallback_pollset(ChkioDrvData *cddp) } static void -stop_driver_event(ChkioDrvData *cddp) -{ - if (cddp->test_data) { - ChkioDriverEvent *cdep = cddp->test_data; - cddp->test_data = NULL; - - if (cdep->in_fd >= 0) { - driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->in_fd, NULL); - close(cdep->in_fd); - } - if (cdep->out_fd >= 0) { - driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->out_fd, NULL); - close(cdep->out_fd); - } - driver_free(cdep); - } -} - -static void stop_fd_change(ChkioDrvData *cddp) { if (cddp->test_data) { @@ -305,14 +271,6 @@ stop_steal(ChkioDrvData *cddp) (ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[1], DO_WRITE, 0); - if (csp->driver_event_fds[0] >= 0) - driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0], - NULL); - if (csp->driver_event_fds[1] >= 0) - driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1], - NULL); driver_free(csp); } } @@ -327,10 +285,6 @@ stop_steal_aux(ChkioDrvData *cddp) close(csap->driver_select_fds[0]); if (csap->driver_select_fds[1] >= 0) close(csap->driver_select_fds[1]); - if (csap->driver_event_fds[0] >= 0) - close(csap->driver_event_fds[0]); - if (csap->driver_event_fds[1] >= 0) - close(csap->driver_event_fds[1]); driver_free(csap); } } @@ -354,10 +308,13 @@ static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port) abort(); } case Selected: - driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 0); - /*fall through*/ case Opened: - close(pip->read_fd); + TRACEF(("%T: Close pipe [%d->%d]\n", driver_mk_port(port), pip->write_fd, + pip->read_fd)); + if (pip->wasSelected) + driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ|ERL_DRV_USE, 0); + else + close(pip->read_fd); close(pip->write_fd); pip->state = Closed; break; @@ -445,9 +402,6 @@ chkio_drv_stop(ErlDrvData drv_data) { case CHKIO_BAD_FD_IN_POLLSET: stop_bad_fd_in_pollset(cddp); break; - case CHKIO_DRIVER_EVENT: - stop_driver_event(cddp); - break; case CHKIO_FD_CHANGE: stop_fd_change(cddp); break; @@ -557,6 +511,9 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event) driver_failure_atom(cddp->port, "input_fd_not_found"); break; } + case CHKIO_FD_CHANGE: + /* This may be triggered when an fd is closed while being selected on. */ + break; case CHKIO_STEAL: break; case CHKIO_STEAL_AUX: @@ -621,55 +578,6 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event) } static void -chkio_drv_ready_event(ErlDrvData drv_data, - ErlDrvEvent event, - ErlDrvEventData event_data) -{ -#ifdef UNIX - ChkioDrvData *cddp = (ChkioDrvData *) drv_data; - switch (cddp->test) { - case CHKIO_DRIVER_EVENT: { -#ifdef HAVE_POLL_H - ChkioDriverEvent *cdep = cddp->test_data; - int fd = (int) (ErlDrvSInt) event; - if (fd == cdep->in_fd) { - if (event_data->events == POLLIN - && event_data->revents == POLLIN) { - cdep->in_ok++; - } - else { - driver_failure_atom(cddp->port, "invalid_input_fd_events"); - } - break; - } - if (fd == cdep->out_fd) { - if (event_data->events == POLLOUT - && event_data->revents == POLLOUT) { - cdep->out_ok++; - } - else { - driver_failure_atom(cddp->port, "invalid_output_fd_events"); - } - break; - } -#endif - } - case CHKIO_STEAL: -#ifdef HAVE_POLL_H - break; -#endif - case CHKIO_STEAL_AUX: -#ifdef HAVE_POLL_H - break; -#endif - default: - driver_failure_atom(cddp->port, "unexpected_ready_event"); - break; - } -#endif /* UNIX */ -} - -static void chkio_drv_timeout(ErlDrvData drv_data) { #ifdef UNIX @@ -779,25 +687,6 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; stop_bad_fd_in_pollset(cddp); break; - case CHKIO_DRIVER_EVENT: { - ChkioDriverEvent *cdep = cddp->test_data; - if (!cdep->in_ok || !cdep->out_ok) { - if (!cdep->in_ok) - driver_failure_atom(cddp->port, "got_no_input_events"); - if (!cdep->out_ok) - driver_failure_atom(cddp->port, "got_no_output_events"); - } - else { - char *c = driver_alloc(sizeof(char)*2*30); - if (!c) - driver_failure_posix(cddp->port, ENOMEM); - *rbuf = c; - res_len = sprintf(c, "in=%d\nout=%d\n", - cdep->in_ok, cdep->out_ok); - } - stop_driver_event(cddp); - break; - } case CHKIO_FD_CHANGE: { ChkioFdChange *cfcp = cddp->test_data; if (!cfcp->same_fd) @@ -937,69 +826,6 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; break; } - case CHKIO_DRIVER_EVENT: { -#ifndef HAVE_POLL_H - res_str = "skip: Need the poll.h header for this test, but it doesn't exist"; - res_len = -1; -#else /* HAVE_POLL_H */ - int in_fd = open("/dev/zero", O_RDONLY); - int out_fd = open("/dev/null", O_WRONLY); - - if (in_fd < 0 || out_fd < 0) { - if (in_fd >= 0) - close(in_fd); - if (out_fd >= 0) - close(out_fd); - driver_failure_posix(cddp->port, errno); - } - else { - ChkioDriverEvent *cdep = driver_alloc(sizeof(ChkioDriverEvent)); - if (!cdep) - driver_failure_posix(cddp->port, ENOMEM); - else { - int res; - cddp->test_data = cdep; - - cdep->in_fd = in_fd; - cdep->in_data.events = POLLIN; - cdep->in_data.revents = 0; - cdep->in_ok = 0; - - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) in_fd, - &cdep->in_data); - if (res < 0) { - res_str = "skip: driver_event() not supported"; - res_len = -1; - close(in_fd); - close(out_fd); - cdep->in_fd = -1; - cdep->out_fd = -1; - } - else { - res_str = "ok"; - res_len = -1; - - cdep->out_fd = out_fd; - cdep->out_data.events = POLLOUT; - cdep->out_data.revents = 0; - cdep->out_ok = 0; - - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) out_fd, - &cdep->out_data); - if (res < 0) { - close(out_fd); - cdep->out_fd = -1; - driver_failure_atom(cddp->port, "driver_event_failed"); - } - } - - } - } -#endif /* HAVE_POLL_H */ - break; - } case CHKIO_FD_CHANGE: { ChkioFdChange *cfcp = driver_alloc(sizeof(ChkioFdChange)); if (!cfcp) @@ -1028,58 +854,19 @@ chkio_drv_control(ErlDrvData drv_data, res_len = -1; } else { - int driver_event_fds[2]; int driver_select_fds[2]; cddp->test_data = csp; memcpy(c, buf, len); c[len] = '\0'; if (sscanf(c, - "fds:%d:%d:%d:%d", + "fds:%d:%d", &driver_select_fds[0], - &driver_select_fds[1], - &driver_event_fds[0], - &driver_event_fds[1]) != 4) - driver_failure_atom(cddp->port, "bad_input"); + &driver_select_fds[1]) != 2) + driver_failure_atom(cddp->port, "bad_input"); else { int res = 0; - if (driver_event_fds[0] < 0) { /* Have no working driver_event() ... */ - csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ - csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */ - csp->driver_event_fds[0] = -1; - csp->driver_event_fds[1] = -1; - } - else { /* Have working driver_event() ... */ -#ifndef HAVE_POLL_H - driver_failure_atom(cddp->port, "unexpected_result"); - res = -1; -#else - csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ - csp->driver_event_fds[1] = driver_select_fds[1]; /* Out */ - csp->driver_event_fds[0] = driver_event_fds[0]; /* In */ - csp->driver_select_fds[1] = driver_event_fds[1]; /* Out */ - - /* Steal with driver_event() */ - - csp->event_data[0].events = POLLIN; - csp->event_data[0].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0], - &csp->event_data[0]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed_to_steal"); - if (res >= 0) { - csp->event_data[1].events = POLLOUT; - csp->event_data[1].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1], - &csp->event_data[1]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed_to_steal"); - } -#endif - } + csp->driver_select_fds[0] = driver_select_fds[0]; /* In */ + csp->driver_select_fds[1] = driver_select_fds[1]; /* Out */ /* Steal with driver_select() */ if (res >= 0) { @@ -1109,37 +896,17 @@ chkio_drv_control(ErlDrvData drv_data, break; } case CHKIO_STEAL_AUX: { - int read_fds[2]; - int write_fds[2]; + int read_fd; + int write_fd; - read_fds[0] = open("/dev/zero", O_RDONLY); - write_fds[0] = open("/dev/null", O_WRONLY); - -#ifdef HAVE_POLL_H - read_fds[1] = open("/dev/zero", O_RDONLY); - write_fds[1] = open("/dev/null", O_WRONLY); -#else - read_fds[1] = -1; - write_fds[1] = -1; -#endif + read_fd = open("/dev/zero", O_RDONLY); + write_fd = open("/dev/null", O_WRONLY); - if (read_fds[0] < 0 - || write_fds[0] < 0 -#ifdef HAVE_POLL_H - || read_fds[1] < 0 - || write_fds[1] < 0 -#endif - ) { - if (read_fds[0] < 0) - close(read_fds[0]); - if (write_fds[0] < 0) - close(write_fds[0]); -#ifdef HAVE_POLL_H - if (read_fds[1] < 0) - close(read_fds[1]); - if (write_fds[1] < 0) - close(write_fds[1]); -#endif + if (read_fd < 0 || write_fd < 0) { + if (read_fd < 0) + close(read_fd); + if (write_fd < 0) + close(write_fd); driver_failure_posix(cddp->port, errno); } else { @@ -1153,11 +920,8 @@ chkio_drv_control(ErlDrvData drv_data, int res; cddp->test_data = csap; - csap->driver_select_fds[0] = read_fds[0]; - csap->driver_select_fds[1] = write_fds[0]; - - csap->driver_event_fds[0] = read_fds[1]; - csap->driver_event_fds[1] = write_fds[1]; + csap->driver_select_fds[0] = read_fd; + csap->driver_select_fds[1] = write_fd; res = driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) csap->driver_select_fds[0], @@ -1173,32 +937,6 @@ chkio_drv_control(ErlDrvData drv_data, if (res < 0) driver_failure_atom(cddp->port, "driver_select_failed"); } -#ifdef HAVE_POLL_H - if (res >= 0) { - csap->event_data[0].events = POLLIN; - csap->event_data[0].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[0], - &csap->event_data[0]); - if (res < 0) { - close(csap->driver_event_fds[0]); - csap->driver_event_fds[0] = -1; - close(csap->driver_event_fds[1]); - csap->driver_event_fds[1] = -1; - res = 0; - } - else { - csap->event_data[1].events = POLLOUT; - csap->event_data[1].revents = 0; - res = driver_event(cddp->port, - (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[1], - &csap->event_data[1]); - if (res < 0) - driver_failure_atom(cddp->port, - "driver_event_failed"); - } - } -#endif if (res < 0) { res_str = "error"; res_len = -1; @@ -1213,11 +951,9 @@ chkio_drv_control(ErlDrvData drv_data, else { *rbuf = c; res_len = sprintf(c, - "fds:%d:%d:%d:%d", + "fds:%d:%d", csap->driver_select_fds[0], - csap->driver_select_fds[1], - csap->driver_event_fds[0], - csap->driver_event_fds[1]); + csap->driver_select_fds[1]); } } } @@ -1257,7 +993,7 @@ chkio_drv_control(ErlDrvData drv_data, } TRACEF(("%T: Created pipe [%d->%d]\n", cddp->id, fds[1], fds[0])); pip->read_fd = fds[0]; - pip->write_fd = fds[1]; + pip->write_fd = fds[1]; pip->state = Opened; pip->wasSelected = 0; pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024; @@ -1267,7 +1003,8 @@ chkio_drv_control(ErlDrvData drv_data, }/*fall through*/ case Opened: { if (op & 1) { - TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id, pip->next_write, pip->write_fd, pip->read_fd)); + TRACEF(("%T: Write %d to opened pipe [%d->%d]\n", cddp->id, + pip->next_write, pip->write_fd, pip->read_fd)); if (write(pip->write_fd, &pip->next_write, sizeof(int)) != sizeof(int)) { fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno); abort(); @@ -1276,8 +1013,12 @@ chkio_drv_control(ErlDrvData drv_data, } op >>= 1; if (pip->wasSelected && (op & 1)) { - TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd)); - if (close(pip->read_fd) || close(pip->write_fd)) { + TRACEF(("%T: Close pipe [%d->%d]\n", cddp->id, pip->write_fd, + pip->read_fd)); + drv_use_singleton.fd_stop_select = -2; /* disable stop_select asserts */ + if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, + DO_READ|ERL_DRV_USE, 0) + || close(pip->write_fd)) { fprintf(stderr, "Failed to close pipe, errno=%d\n", errno); abort(); } @@ -1285,8 +1026,10 @@ chkio_drv_control(ErlDrvData drv_data, break; } else { - TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd)); - if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 1)) { + TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, + pip->write_fd, pip->read_fd)); + if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, + DO_READ|ERL_DRV_USE, 1)) { fprintf(stderr, "driver_select failed for fd=%d\n", pip->read_fd); abort(); } @@ -1294,13 +1037,13 @@ chkio_drv_control(ErlDrvData drv_data, pip->wasSelected = 1; op >>= 1; if (pip->next_write != pip->next_read) { /* pipe not empty */ - if (op & 1) { + if (op & 1) { pip->state = Waiting; /* Wait for reader */ break; } op >>= 1; } - } + } }/*fall through*/ case Selected: if (op & 1) { @@ -1329,7 +1072,7 @@ chkio_drv_control(ErlDrvData drv_data, fprintf(stderr, "Failed to write to pipe fd=%d, errno=%d\n", pip->write_fd, errno); abort(); } - pip->next_write++; + pip->next_write++; } break; case Waiting: @@ -1583,7 +1326,12 @@ static void chkio_drv_stop_select(ErlDrvEvent e, void* null) if (!(drv_use_singleton.fd_stop_select < 0)) { assert_print("fd_stop_select<0", __LINE__); abort(); } - drv_use_singleton.fd_stop_select = (int)(long)e; + /* fd_stop_select counting is disabled if this is set to -2 */ + if (drv_use_singleton.fd_stop_select == -2) { + TRACEF(("closing %d\n", (int)(long)e)); + close((int)(long)e); + } else + drv_use_singleton.fd_stop_select = (int)(long)e; /* Can't call chkio_drv_use directly here. That could even be recursive. * Next timeout will detect it instead. */ diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c index d87c2bec93..fa58e9d5ec 100644 --- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c +++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c @@ -25,8 +25,7 @@ * - ready_input(), * - ready_output(), * - timeout(), - * - driver_async() -> read_async(), and - * - event() + * - driver_async() -> read_async() */ #ifndef UNIX @@ -65,11 +64,9 @@ typedef enum { IOQ_EXIT_READY_OUTPUT = 2, IOQ_EXIT_TIMEOUT = 3, IOQ_EXIT_READY_ASYNC = 4, - IOQ_EXIT_EVENT = 5, IOQ_EXIT_READY_INPUT_ASYNC = 6, IOQ_EXIT_READY_OUTPUT_ASYNC = 7, IOQ_EXIT_TIMEOUT_ASYNC = 8, - IOQ_EXIT_EVENT_ASYNC = 9 } IOQExitTest; typedef struct { @@ -80,9 +77,6 @@ typedef struct { int outstanding_async_task; long async_task; ErlDrvPDL pdl; -#ifdef HAVE_POLL_H - struct erl_drv_event_data event_data; -#endif } IOQExitDrvData; #define EV2FD(EV) ((int) ((long) (EV))) @@ -97,8 +91,6 @@ static ErlDrvSSizeT control(ErlDrvData, unsigned int, static void timeout(ErlDrvData drv_data); static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data); static void flush(ErlDrvData drv_data); -static void event(ErlDrvData drv_data, ErlDrvEvent event, - ErlDrvEventData event_data); static void async_invoke(void*); static void do_driver_async(IOQExitDrvData *); @@ -118,7 +110,7 @@ static ErlDrvEntry ioq_exit_drv_entry = { ready_async, flush, NULL /* call */, - event, + NULL /* unused_event_callback*/, ERL_DRV_EXTENDED_MARKER, ERL_DRV_EXTENDED_MAJOR_VERSION, ERL_DRV_EXTENDED_MINOR_VERSION, @@ -149,10 +141,6 @@ start(ErlDrvPort port, char *command) ddp->outstanding_async_task = 0; ddp->async_task = -1; ddp->pdl = driver_pdl_create(port); -#ifdef HAVE_POLL_H - ddp->event_data.events = (short) 0; - ddp->event_data.revents = (short) 0; -#endif return (ErlDrvData) ddp; } @@ -192,27 +180,6 @@ static ErlDrvSSizeT control(ErlDrvData drv_data, #else goto done; #endif - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: -#ifdef UNIX -#ifdef HAVE_POLL_H - ddp->ofd = open("/dev/null", O_WRONLY); - if (ddp->ofd < 0) { - driver_failure_posix(ddp->port, errno); - return 0; - } - else if (driver_event(ddp->port, FD2EV(ddp->ofd), NULL) != 0) { - res_str = "skip: driver_event() not supported"; - goto done; - } -#else - res_str = "skip: No poll.h found which is needed for this test"; - goto done; -#endif - break; -#else /* UNIX */ - goto done; -#endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: break; @@ -266,13 +233,6 @@ static void stop(ErlDrvData drv_data) close(ddp->ofd); } break; - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: - if (ddp->ofd >= 0) { - driver_event(ddp->port, FD2EV(ddp->ofd), NULL); - close(ddp->ofd); - } - break; #endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: @@ -302,13 +262,6 @@ static void flush(ErlDrvData drv_data) case IOQ_EXIT_READY_OUTPUT_ASYNC: driver_select(ddp->port, FD2EV(ddp->ofd), DO_WRITE, 1); break; - case IOQ_EXIT_EVENT: - case IOQ_EXIT_EVENT_ASYNC: -#ifdef HAVE_POLL_H - ddp->event_data.events |= POLLOUT; - driver_event(ddp->port, FD2EV(ddp->ofd), &ddp->event_data); -#endif - break; #endif case IOQ_EXIT_TIMEOUT: case IOQ_EXIT_TIMEOUT_ASYNC: @@ -395,30 +348,6 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data) } } -static void event(ErlDrvData drv_data, - ErlDrvEvent event, - ErlDrvEventData event_data) -{ - IOQExitDrvData *ddp = (IOQExitDrvData *) drv_data; - - PRINTF(("event(%p, %d, %p) called\r\n", drv_data, EV2FD(event), event_data)); - -#if defined(UNIX) && defined(HAVE_POLL_H) - if (ddp->ofd == EV2FD(event)) { - driver_event(ddp->port, FD2EV(ddp->ofd), NULL); - close(ddp->ofd); - ddp->ofd = -1; - if (ddp->test == IOQ_EXIT_EVENT_ASYNC) - do_driver_async(ddp); - else { - driver_pdl_lock(ddp->pdl); - driver_deq(ddp->port, 1); - driver_pdl_unlock(ddp->pdl); - } - } -#endif -} - static void async_invoke(void *arg) { PRINTF(("async_invoke(%p) called\r\n", arg)); diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c index e7480d2e00..14838f0377 100644 --- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c +++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c @@ -41,10 +41,6 @@ typedef struct { int ofd; int ifd; - int efd; -#ifdef HAVE_POLL_H - struct erl_drv_event_data edata; -#endif } mcd_data_t; static ErlDrvData start(ErlDrvPort port, char *command); @@ -90,7 +86,6 @@ start(ErlDrvPort port, char *command) mcd->ofd = -1; mcd->ifd = -1; - mcd->efd = -1; #ifdef UNIX @@ -105,15 +100,6 @@ start(ErlDrvPort port, char *command) goto error; if (driver_select(port, (ErlDrvEvent) (long) mcd->ifd, DO_READ, 1) != 0) goto error; - -#ifdef HAVE_POLL_H - mcd->efd = open("/dev/null", O_WRONLY); - if (mcd->efd < 0) - goto error; - mcd->edata.events = POLLOUT; - mcd->edata.revents = 0; - driver_event(port, (ErlDrvEvent) (long) mcd->efd, &mcd->edata); -#endif #endif driver_set_timer(port, 0); @@ -135,10 +121,6 @@ stop(ErlDrvData data) close(mcd->ofd); if (mcd->ifd >= 0) close(mcd->ifd); -#ifdef HAVE_POLL_H - if (mcd->efd >= 0) - close(mcd->efd); -#endif #endif driver_free(mcd); } diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index f0e1bcf04b..08d5597d78 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -19,16 +19,20 @@ -module(efile_SUITE). -export([all/0, suite/0]). --export([iter_max_files/1, async_dist/1]). +-export([async_dist/1, + iter_max_files/1, + proc_zero_sized_files/1 + ]). -export([do_iter_max_files/2, do_async_dist/1]). -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [iter_max_files, async_dist]. + [iter_max_files, async_dist, proc_zero_sized_files]. do_async_dist(Dir) -> X = 100, @@ -162,3 +166,44 @@ open_files(Name) -> % io:format("Error reason: ~p", [_Reason]), [] end. + +%% @doc If /proc filesystem exists (no way to know if it is real proc or just +%% a /proc directory), let's read some zero sized files 500 times each, while +%% ensuring that response isn't empty << >> +proc_zero_sized_files(Config) when is_list(Config) -> + {Type, Flavor} = os:type(), + %% Some files which exist on Linux but might be missing on other systems + Inputs = ["/proc/cpuinfo", + "/proc/meminfo", + "/proc/partitions", + "/proc/swaps", + "/proc/version", + "/proc/uptime", + %% curproc is present on freebsd + "/proc/curproc/cmdline"], + case filelib:is_dir("/proc") of + false -> {skip, "/proc not found"}; % skip the test if no /proc + _ when Type =:= unix andalso Flavor =:= sunos -> + %% SunOS has a /proc, but no zero sized special files + {skip, "sunos does not have any zero sized special files"}; + true -> + %% Take away files which do not exist in proc + Inputs1 = lists:filter(fun filelib:is_file/1, Inputs), + + %% Fail if none of mentioned files exist in /proc, did we just get + %% a normal /proc directory without any special files? + ?assertNotEqual([], Inputs1), + + %% For 6 inputs and 500 attempts each this do run anywhere + %% between 500 and 3000 function calls. + lists:foreach( + fun(Filename) -> do_proc_zero_sized(Filename, 500) end, + Inputs1) + end. + +%% @doc Test one file N times to also trigger possible leaking fds and memory +do_proc_zero_sized(_Filename, 0) -> ok; +do_proc_zero_sized(Filename, N) -> + Data = file:read_file(Filename), + ?assertNotEqual(<<>>, Data), + do_proc_zero_sized(Filename, N-1). diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec index b2d0de8835..fc98ba6823 100644 --- a/erts/emulator/test/emulator_smoke.spec +++ b/erts/emulator/test/emulator_smoke.spec @@ -7,3 +7,4 @@ [consistency],"Not reliable in October and March"}. {cases,'Dir',crypto_SUITE,[t_md5]}. {cases,'Dir',float_SUITE,[fpe,cmp_integer]}. +{cases,'Dir',erts_debug_SUITE,[df]}. diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index 5622cce980..d8c5b663e3 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -533,7 +533,7 @@ freeze_node(Node, MS) -> fun () -> erts_debug:set_internal_state(available_internal_state, true), - dport_send(Freezer, DoingIt), + dctrl_dop_send(Freezer, DoingIt), receive after Own -> ok end, erts_debug:set_internal_state(block, MS+Own) end), @@ -544,20 +544,22 @@ make_busy(Node, Time) when is_integer(Time) -> Own = 500, freeze_node(Node, Time+Own), Data = busy_data(), + DCtrl = dctrl(Node), %% first make port busy Pid = spawn_link(fun () -> forever(fun () -> - dport_reg_send(Node, - '__noone__', - Data) + dctrl_dop_reg_send(Node, + '__noone__', + Data) end) end), receive after Own -> ok end, wait_until(fun () -> - case process_info(Pid, status) of - {status, suspended} -> true; - _ -> false - end + case {DCtrl, process_info(Pid, status)} of + {DPrt, {status, suspended}} when is_port(DPrt) -> true; + {DPid, {status, waiting}} when is_pid(DPid) -> true; + _ -> false + end end), %% then dist entry make_busy(Node, [nosuspend], Data), @@ -1048,42 +1050,45 @@ stop_node(Node) -> -define(DOP_DEMONITOR_P, 20). -define(DOP_MONITOR_P_EXIT, 21). -dport_send(To, Msg) -> - Node = node(To), - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_SEND, - ?COOKIE, - To}), - dmsg_ext(Msg)]). - -dport_reg_send(Node, Name, Msg) -> - DPrt = case dport(Node) of - undefined -> - pong = net_adm:ping(Node), - dport(Node); - Prt -> - Prt - end, - port_command(DPrt, [dmsg_hdr(), - dmsg_ext({?DOP_REG_SEND, - self(), - ?COOKIE, - Name}), - dmsg_ext(Msg)]). - -dport(Node) when is_atom(Node) -> +ensure_dctrl(Node) -> + case dctrl(Node) of + undefined -> + pong = net_adm:ping(Node), + dctrl(Node); + DCtrl -> + DCtrl + end. + +dctrl_send(DPrt, Data) when is_port(DPrt) -> + port_command(DPrt, Data); +dctrl_send(DPid, Data) when is_pid(DPid) -> + Ref = make_ref(), + DPid ! {send, self(), Ref, Data}, + receive {Ref, Res} -> Res end. + +dctrl_dop_send(To, Msg) -> + dctrl_send(ensure_dctrl(node(To)), + [dmsg_hdr(), + dmsg_ext({?DOP_SEND, + ?COOKIE, + To}), + dmsg_ext(Msg)]). + +dctrl_dop_reg_send(Node, Name, Msg) -> + dctrl_send(ensure_dctrl(Node), + [dmsg_hdr(), + dmsg_ext({?DOP_REG_SEND, + self(), + ?COOKIE, + Name}), + dmsg_ext(Msg)]). + +dctrl(Node) when is_atom(Node) -> case catch erts_debug:get_internal_state(available_internal_state) of true -> true; _ -> erts_debug:set_internal_state(available_internal_state, true) end, - erts_debug:get_internal_state({dist_port, Node}). + erts_debug:get_internal_state({dist_ctrl, Node}). dmsg_hdr() -> [131, % Version Magic diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl index 8b336b366d..c9c1867049 100644 --- a/erts/emulator/test/estone_SUITE.erl +++ b/erts/emulator/test/estone_SUITE.erl @@ -20,7 +20,7 @@ -module(estone_SUITE). %% Test functions -export([all/0, suite/0, groups/0, - estone/1, estone_bench/1]). + estone/1, estone_bench/1, pgo/0]). %% Internal exports for EStone tests -export([lists/1, @@ -44,9 +44,9 @@ links/1,lproc/1, run_micro/3,p1/1,ppp/3,macro/2,micros/0]). - --include_lib("common_test/include/ct.hrl"). +-ifndef(PGO). -include_lib("common_test/include/ct_event.hrl"). +-endif. %% EStone defines -define(TOTAL, (3000 * 1000 * 100)). %% 300 secs @@ -85,13 +85,28 @@ estone(Config) when is_list(Config) -> estone_bench(Config) -> DataDir = proplists:get_value(data_dir,Config), L = ?MODULE:macro(?MODULE:micros(),DataDir), - [ct_event:notify( - #event{name = benchmark_data, - data = [{name,proplists:get_value(title,Mark)}, - {value,proplists:get_value(estones,Mark)}]}) - || Mark <- L], + {Total, Stones} = sum_micros(L, 0, 0), + notify([[{title,"ESTONES"}, {estones, Stones}] | L]), L. +-ifndef(PGO). +notify(Marks) -> + [ct_event:notify( + #event{name = benchmark_data, + data = [{name,proplists:get_value(title, Mark)}, + {value,proplists:get_value(estones, Mark)}]}) + || Mark <- Marks]. +-else. +notify(_) -> + ok. +-endif. + +%% The benchmarks to run in order to guide PGO (profile guided optimisation) +pgo() -> + %% We run all benchmarks except the port_io as we don't want to + %% have to build a custom port. + Micros = ?MODULE:micros() -- [micro(port_io)], + ?MODULE:macro(Micros,[]). %% %% Calculate CPU speed @@ -364,7 +379,7 @@ monotonic_time() -> try erlang:monotonic_time() catch error:undef -> erlang:now() end. subtr(Before, After) when is_integer(Before), is_integer(After) -> - erlang:convert_time_unit(After-Before, native, microsecond); + erlang:convert_time_unit(After-Before, native, 1000000); subtr({_,_,_}=Before, {_,_,_}=After) -> timer:now_diff(After, Before). diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index aaca522da6..0f27251fcb 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -21,7 +21,7 @@ -module(exception_SUITE). -export([all/0, suite/0, - badmatch/1, pending_errors/1, nil_arith/1, + badmatch/1, pending_errors/1, nil_arith/1, top_of_stacktrace/1, stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1, exception_with_heap_frag/1, line_numbers/1]). @@ -36,8 +36,8 @@ suite() -> {timetrap, {minutes, 1}}]. all() -> - [badmatch, pending_errors, nil_arith, stacktrace, - nested_stacktrace, raise, gunilla, per, + [badmatch, pending_errors, nil_arith, top_of_stacktrace, + stacktrace, nested_stacktrace, raise, gunilla, per, exception_with_heap_frag, line_numbers]. -define(try_match(E), @@ -241,7 +241,54 @@ ba_bnot(A) -> io:format("bnot ~p", [A]), {'EXIT', {badarith, _}} = (catch bnot A). +%% Test that BIFs are added to the top of the stacktrace. + +top_of_stacktrace(Conf) when is_list(Conf) -> + %% Arithmetic operators + {'EXIT', {badarith, [{erlang, '+', [1, ok], _} | _]}} = (catch my_add(1, ok)), + {'EXIT', {badarith, [{erlang, '-', [1, ok], _} | _]}} = (catch my_minus(1, ok)), + {'EXIT', {badarith, [{erlang, '*', [1, ok], _} | _]}} = (catch my_times(1, ok)), + {'EXIT', {badarith, [{erlang, 'div', [1, ok], _} | _]}} = (catch my_div(1, ok)), + {'EXIT', {badarith, [{erlang, 'div', [1, 0], _} | _]}} = (catch my_div(1, 0)), + {'EXIT', {badarith, [{erlang, 'rem', [1, ok], _} | _]}} = (catch my_rem(1, ok)), + {'EXIT', {badarith, [{erlang, 'rem', [1, 0], _} | _]}} = (catch my_rem(1, 0)), + + %% Bit operators + {'EXIT', {badarith, [{erlang, 'band', [1, ok], _} | _]}} = (catch my_band(1, ok)), + {'EXIT', {badarith, [{erlang, 'bor', [1, ok], _} | _]}} = (catch my_bor(1, ok)), + {'EXIT', {badarith, [{erlang, 'bsl', [1, ok], _} | _]}} = (catch my_bsl(1, ok)), + {'EXIT', {badarith, [{erlang, 'bsr', [1, ok], _} | _]}} = (catch my_bsr(1, ok)), + {'EXIT', {badarith, [{erlang, 'bxor', [1, ok], _} | _]}} = (catch my_bxor(1, ok)), + {'EXIT', {badarith, [{erlang, 'bnot', [ok], _} | _]}} = (catch my_bnot(ok)), + + %% Tuples + {'EXIT', {badarg, [{erlang, element, [1, ok], _} | _]}} = (catch my_element(1, ok)), + {'EXIT', {badarg, [{erlang, element, [ok, {}], _} | _]}} = (catch my_element(ok, {})), + {'EXIT', {badarg, [{erlang, element, [1, {}], _} | _]}} = (catch my_element(1, {})), + {'EXIT', {badarg, [{erlang, element, [1, {}], _} | _]}} = (catch element(1, erlang:make_tuple(0, ok))), + + %% System limits + Maxbig = maxbig(), + MinusMaxbig = -Maxbig, + {'EXIT', {system_limit, [{erlang, '+', [Maxbig, 1], _} | _]}} = (catch my_add(Maxbig, 1)), + {'EXIT', {system_limit, [{erlang, '+', [Maxbig, 1], _} | _]}} = (catch my_add(maxbig_gc(), 1)), + {'EXIT', {system_limit, [{erlang, '-', [MinusMaxbig, 1], _} | _]}} = (catch my_minus(-Maxbig, 1)), + {'EXIT', {system_limit, [{erlang, '-', [MinusMaxbig, 1], _} | _]}} = (catch my_minus(-maxbig_gc(), 1)), + {'EXIT', {system_limit, [{erlang, '*', [Maxbig, 2], _} | _]}} = (catch my_times(Maxbig, 2)), + {'EXIT', {system_limit, [{erlang, '*', [Maxbig, 2], _} | _]}} = (catch my_times(maxbig_gc(), 2)), + {'EXIT', {system_limit, [{erlang, 'bnot', [Maxbig], _} | _]}} = (catch my_bnot(Maxbig)), + {'EXIT', {system_limit, [{erlang, 'bnot', [Maxbig], _} | _]}} = (catch my_bnot(maxbig_gc())), + ok. + +maxbig() -> + %% We assume that the maximum arity is (1 bsl 19) - 1. + Ws = erlang:system_info(wordsize), + (((1 bsl ((16777184 * (Ws div 4))-1)) - 1) bsl 1) + 1. +maxbig_gc() -> + Maxbig = maxbig(), + erlang:garbage_collect(), + Maxbig. stacktrace(Conf) when is_list(Conf) -> Tag = make_ref(), @@ -253,9 +300,9 @@ stacktrace(Conf) when is_list(Conf) -> St1 = erase(stacktrace1), St1 = erase(stacktrace2), St1 = erlang:get_stacktrace(), - {caught2,{error,badarith},[{?MODULE,my_add,2,_}|_]=St2} = + {caught2,{error,badarith},[{erlang,'+',[0,a],_},{?MODULE,my_add,2,_}|_]=St2} = stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}), - [{?MODULE,my_div,2,_}|_] = erase(stacktrace1), + [{erlang,'div',[1,0],_},{?MODULE,my_div,2,_}|_] = erase(stacktrace1), St2 = erase(stacktrace2), St2 = erlang:get_stacktrace(), {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3,_}|_]=St3} = @@ -308,13 +355,13 @@ nested_stacktrace(Conf) when is_list(Conf) -> nested_stacktrace_1({{value,{V,x1}},void,{V,x1}}, {void,void,void}), {caught1, - [{?MODULE,my_add,2,_}|_], + [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_], value2, - [{?MODULE,my_add,2,_}|_]} = + [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_]} = nested_stacktrace_1({{'add',{V,x1}},error,badarith}, {{value,{V,x2}},void,{V,x2}}), {caught1, - [{?MODULE,my_add,2,_}|_], + [{erlang,'+',[V,x1],_},{?MODULE,my_add,2,_}|_], {caught2,[{erlang,abs,[V],_}|_]}, [{erlang,abs,[V],_}|_]} = nested_stacktrace_1({{'add',{V,x1}},error,badarith}, @@ -355,7 +402,7 @@ raise(Conf) when is_list(Conf) -> end, A = erlang:get_stacktrace(), A = get(raise), - [{?MODULE,my_div,2,_}|_] = A, + [{erlang,'div',[1, 0], _},{?MODULE,my_div,2,_}|_] = A, %% N = 8, % Must be even N = erlang:system_flag(backtrace_depth, N), @@ -404,11 +451,20 @@ foo({raise,{Class,Reason,Stacktrace}}) -> erlang:raise(Class, Reason, Stacktrace). %%foo(function_clause) -> % must not be defined! -my_div(A, B) -> - A div B. +my_add(A, B) -> A + B. +my_minus(A, B) -> A - B. +my_times(A, B) -> A * B. +my_div(A, B) -> A div B. +my_rem(A, B) -> A rem B. + +my_band(A, B) -> A band B. +my_bor(A, B) -> A bor B. +my_bsl(A, B) -> A bsl B. +my_bsr(A, B) -> A bsr B. +my_bxor(A, B) -> A bxor B. +my_bnot(A) -> bnot A. -my_add(A, B) -> - A + B. +my_element(A, B) -> element(A, B). my_abs(X) -> abs(X). @@ -606,6 +662,15 @@ line_numbers(Config) when is_list(Config) -> {?MODULE,line_numbers,1,_}|_]}} = (catch applied_bif_2()), + {'EXIT',{badarith, + [{?MODULE,increment1,1,[{file,"increment.erl"},{line,45}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch increment1(x)), + {'EXIT',{badarith, + [{?MODULE,increment2,1,[{file,"increment.erl"},{line,48}]}, + {?MODULE,line_numbers,1,_}|_]}} = + (catch increment2(x)), + ok. id(I) -> I. @@ -706,3 +771,15 @@ applied_bif_2() -> %Line 8 R = process_info(self(), current_location), %Line 9 fail = R, %Line 10 ok. %Line 11 + +%% The increment instruction used to decrement the instruction +%% pointer, which would cause the line number in a stack trace to +%% be the previous line number. + +-file("increment.erl", 42). +increment1(Arg) -> %Line 43 + Res = id(Arg), %Line 44 + Res + 1. %Line 45 +increment2(Arg) -> %Line 46 + _ = id(Arg), %Line 47 + Arg + 1. %Line 48 diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index e4640909aa..7d29ebec52 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -22,6 +22,7 @@ -export([all/0, suite/0, bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1, + bad_arglist/1, equality/1,ordering/1, fun_to_port/1,t_phash/1,t_phash2/1,md5/1, refc/1,refc_ets/1,refc_dist/1, @@ -39,6 +40,7 @@ suite() -> all() -> [bad_apply, bad_fun_call, badarity, ext_badarity, + bad_arglist, equality, ordering, fun_to_port, t_phash, t_phash2, md5, refc, refc_ets, refc_dist, const_propagation, t_arity, t_is_function2, t_fun_info, @@ -107,6 +109,18 @@ bad_call_fc(Fun) -> ct:fail({bad_result,Other}) end. +% Test erlang:apply with non-proper arg-list +bad_arglist(Config) when is_list(Config) -> + Fun = fun(A,B) -> A+B end, + {'EXIT', {badarg,_}} = (catch apply(Fun, 17)), + {'EXIT', {badarg,_}} = (catch apply(Fun, [17|18])), + {'EXIT', {badarg,_}} = (catch apply(Fun, [17,18|19])), + {'EXIT', {badarg,_}} = (catch apply(lists,seq, 17)), + {'EXIT', {badarg,_}} = (catch apply(lists,seq, [17|18])), + {'EXIT', {badarg,_}} = (catch apply(lists,seq, [17,18|19])), + ok. + + %% Call and apply valid funs with wrong number of arguments. badarity(Config) when is_list(Config) -> diff --git a/erts/emulator/test/iovec_SUITE.erl b/erts/emulator/test/iovec_SUITE.erl new file mode 100644 index 0000000000..49dc64b0d2 --- /dev/null +++ b/erts/emulator/test/iovec_SUITE.erl @@ -0,0 +1,176 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(iovec_SUITE). + +-export([all/0, suite/0, init_per_suite/1, end_per_suite/1]). + +-export([integer_lists/1, binary_lists/1, empty_lists/1, empty_binary_lists/1, + mixed_lists/1, improper_lists/1, illegal_lists/1, cons_bomb/1, + sub_binary_lists/1, iolist_to_iovec_idempotence/1, + iolist_to_iovec_correctness/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap, {minutes, 2}}]. + +all() -> + [integer_lists, binary_lists, empty_lists, empty_binary_lists, mixed_lists, + sub_binary_lists, illegal_lists, improper_lists, cons_bomb, + iolist_to_iovec_idempotence, iolist_to_iovec_correctness]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + application:stop(os_mon), + Config. + +integer_lists(Config) when is_list(Config) -> + Variations = gen_variations([I || I <- lists:seq(1, 255)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +sub_binary_lists(Config) when is_list(Config) -> + Parent = <<0:256/unit:8, "gazurka">>, + <<0:196/unit:8, Child/binary>> = Parent, + equivalence_test(fun erlang:iolist_to_iovec/1, gen_variations(Child)). + +binary_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +empty_lists(Config) when is_list(Config) -> + Variations = gen_variations([[] || _ <- lists:seq(1, 256)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations), + [] = erlang:iolist_to_iovec([]), + ok. + +empty_binary_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<>> || _ <- lists:seq(1, 8192)]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations), + [] = erlang:iolist_to_iovec(Variations), + ok. + +mixed_lists(Config) when is_list(Config) -> + Variations = gen_variations([<<>>, lists:seq(1, 40), <<12, 45, 78>>]), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +illegal_lists(Config) when is_list(Config) -> + BitStrs = gen_variations(["gurka", <<1:1>>, "gaffel"]), + BadInts = gen_variations(["gurka", 890, "gaffel"]), + Atoms = gen_variations([gurka, "gaffel"]), + BadTails = [["test" | 0], ["gurka", gaffel]], + + Variations = + BitStrs ++ BadInts ++ Atoms ++ BadTails, + + illegality_test(fun erlang:iolist_to_iovec/1, Variations). + +improper_lists(Config) when is_list(Config) -> + Variations = [ + [[[[1 | <<2>>] | <<3>>] | <<4>>] | <<5>>], + [[<<1>>, 2] | <<3, 4, 5>>], + [1, 2, 3 | <<4, 5>>] + ], + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +cons_bomb(Config) when is_list(Config) -> + IntBase = gen_variations([I || I <- lists:seq(1, 255)]), + BinBase = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixBase = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Rounds = + case system_mem_size() of + Mem when Mem >= (16 bsl 30) -> 32; + Mem when Mem >= (3 bsl 30) -> 28; + _ -> 20 + end, + + Variations = gen_variations([IntBase, BinBase, MixBase], Rounds), + equivalence_test(fun erlang:iolist_to_iovec/1, Variations). + +iolist_to_iovec_idempotence(Config) when is_list(Config) -> + IntVariations = gen_variations([I || I <- lists:seq(1, 255)]), + BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Variations = [IntVariations, BinVariations, MixVariations], + Optimized = erlang:iolist_to_iovec(Variations), + + true = Optimized =:= erlang:iolist_to_iovec(Optimized), + ok. + +iolist_to_iovec_correctness(Config) when is_list(Config) -> + IntVariations = gen_variations([I || I <- lists:seq(1, 255)]), + BinVariations = gen_variations([<<I:8>> || I <- lists:seq(1, 255)]), + MixVariations = gen_variations([<<12, 45, 78>>, lists:seq(1, 255)]), + + Variations = [IntVariations, BinVariations, MixVariations], + Optimized = erlang:iolist_to_iovec(Variations), + + true = is_iolist_equal(Optimized, Variations), + ok. + +illegality_test(Fun, Variations) -> + [{'EXIT',{badarg, _}} = (catch Fun(Variation)) || Variation <- Variations], + ok. + +equivalence_test(Fun, [Head | _] = Variations) -> + %% Check that each variation is equal to the others, and that the sum of + %% them is equal to the input. + Comparand = Fun(Head), + [true = is_iolist_equal(Comparand, Fun(V)) || V <- Variations], + true = is_iolist_equal(Variations, Fun(Variations)), + ok. + +is_iolist_equal(A, B) -> + iolist_to_binary(A) =:= iolist_to_binary(B). + +%% Generates a bunch of lists whose contents will be equal to Base repeated a +%% few times. The lists only differ by their structure, so their reduction to +%% a simpler format should yield the same result. +gen_variations(Base) -> + gen_variations(Base, 16). +gen_variations(Base, N) -> + [gen_flat_list(Base, N), + gen_nested_list(Base, N), + gen_nasty_list(Base, N)]. + +gen_flat_list(Base, N) -> + lists:flatten(gen_nested_list(Base, N)). + +gen_nested_list(Base, N) -> + [Base || _ <- lists:seq(1, N)]. + +gen_nasty_list(Base, N) -> + gen_nasty_list_1(gen_nested_list(Base, N), []). +gen_nasty_list_1([], Result) -> + Result; +gen_nasty_list_1([Head | Base], Result) when is_list(Head) -> + gen_nasty_list_1(Base, [[Result], [gen_nasty_list_1(Head, [])]]); +gen_nasty_list_1([Head | Base], Result) -> + gen_nasty_list_1(Base, [[Result], [Head]]). + +system_mem_size() -> + application:ensure_all_started(os_mon), + {Tot,_Used,_} = memsup:get_memory_data(), + Tot. diff --git a/erts/emulator/test/lcnt_SUITE.erl b/erts/emulator/test/lcnt_SUITE.erl new file mode 100644 index 0000000000..504b9b54cf --- /dev/null +++ b/erts/emulator/test/lcnt_SUITE.erl @@ -0,0 +1,156 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(lcnt_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export( + [all/0, suite/0, + init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2]). + +-export( + [toggle_lock_counting/1, error_on_invalid_category/1, preserve_locks/1]). + +suite() -> + [{ct_hooks,[ts_install_cth]}, + {timetrap, {seconds, 10}}]. + +all() -> + [toggle_lock_counting, error_on_invalid_category, preserve_locks]. + +init_per_suite(Config) -> + case erlang:system_info(lock_counting) of + true -> + %% The tests will run straight over these properties, so we have to + %% preserve them to avoid tainting the other tests. + OldCopySave = erts_debug:lcnt_control(copy_save), + OldMask = erts_debug:lcnt_control(mask), + [{lcnt_SUITE, {OldCopySave, OldMask}} | Config]; + _ -> + {skip, "Lock counting is not enabled"} + end. + +end_per_suite(Config) -> + {OldCopySave, OldMask} = proplists:get_value(lcnt_SUITE, Config), + + erts_debug:lcnt_control(copy_save, OldCopySave), + OldCopySave = erts_debug:lcnt_control(copy_save), + + erts_debug:lcnt_control(mask, OldMask), + OldMask = erts_debug:lcnt_control(mask), + + erts_debug:lcnt_clear(), + ok. + +init_per_testcase(_Case, Config) -> + disable_lock_counting(), + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +disable_lock_counting() -> + ok = erts_debug:lcnt_control(copy_save, false), + ok = erts_debug:lcnt_control(mask, []), + ok = erts_debug:lcnt_clear(), + + %% Sanity check. + false = erts_debug:lcnt_control(copy_save), + [] = erts_debug:lcnt_control(mask), + + %% The above commands rely on some lazy operations, so we'll have to wait + %% for the list to clear. + ok = wait_for_empty_lock_list(). + +wait_for_empty_lock_list() -> + wait_for_empty_lock_list(10). +wait_for_empty_lock_list(Tries) when Tries > 0 -> + try_flush_cleanup_ops(), + case erts_debug:lcnt_collect() of + [{duration, _}, {locks, []}] -> + ok; + _ -> + timer:sleep(50), + wait_for_empty_lock_list(Tries - 1) + end; +wait_for_empty_lock_list(0) -> + ct:fail("Lock list failed to clear after disabling lock counting."). + +%% Queue up a lot of thread progress cleanup ops in a vain attempt to +%% flush the lock list. +try_flush_cleanup_ops() -> + false = lists:member(process, erts_debug:lcnt_control(mask)), + [spawn(fun() -> ok end) || _ <- lists:seq(1, 1000)]. + +%% +%% Test cases +%% + +toggle_lock_counting(Config) when is_list(Config) -> + Categories = + [allocator, db, debug, distribution, generic, io, process, scheduler], + lists:foreach( + fun(Category) -> + Locks = get_lock_info_for(Category), + if + Locks =/= [] -> + disable_lock_counting(); + Locks =:= [] -> + ct:fail("Failed to toggle ~p locks.", [Category]) + end + end, Categories). + +get_lock_info_for(Categories) when is_list(Categories) -> + ok = erts_debug:lcnt_control(mask, Categories), + [{duration, _}, {locks, Locks}] = erts_debug:lcnt_collect(), + Locks; + +get_lock_info_for(Category) when is_atom(Category) -> + get_lock_info_for([Category]). + +preserve_locks(Config) when is_list(Config) -> + erts_debug:lcnt_control(mask, [process]), + + erts_debug:lcnt_control(copy_save, true), + [spawn(fun() -> ok end) || _ <- lists:seq(1, 1000)], + + %% Wait for the processes to be fully destroyed before disabling copy_save, + %% then remove all active locks from the list. (There's no foolproof method + %% to do this; sleeping before/after is the best way we have) + timer:sleep(500), + + erts_debug:lcnt_control(copy_save, false), + erts_debug:lcnt_control(mask, []), + + try_flush_cleanup_ops(), + timer:sleep(500), + + case erts_debug:lcnt_collect() of + [{duration, _}, {locks, Locks}] when length(Locks) > 0 -> + ct:pal("Preserved ~p locks.", [length(Locks)]); + [{duration, _}, {locks, []}] -> + ct:fail("copy_save didn't preserve any locks.") + end. + +error_on_invalid_category(Config) when is_list(Config) -> + {error, badarg, q_invalid} = erts_debug:lcnt_control(mask, [q_invalid]), + ok. diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl index a012fa1da2..19c3844c40 100644 --- a/erts/emulator/test/lttng_SUITE.erl +++ b/erts/emulator/test/lttng_SUITE.erl @@ -81,7 +81,6 @@ end_per_testcase(Case, _Config) -> %% Not tested yet %% org_erlang_otp:driver_process_exit -%% org_erlang_otp:driver_event %% tracepoints %% @@ -100,7 +99,6 @@ end_per_testcase(Case, _Config) -> %% org_erlang_otp:driver_flush %% org_erlang_otp:driver_stop_select %% org_erlang_otp:driver_timeout -%% org_erlang_otp:driver_event %% org_erlang_otp:driver_ready_output %% org_erlang_otp:driver_ready_input %% org_erlang_otp:driver_output @@ -431,7 +429,6 @@ txt() -> "%% org_erlang_otp:driver_flush\n" "%% org_erlang_otp:driver_stop_select\n" "%% org_erlang_otp:driver_timeout\n" - "%% org_erlang_otp:driver_event\n" "%% org_erlang_otp:driver_ready_output\n" "%% org_erlang_otp:driver_ready_input\n" "%% org_erlang_otp:driver_output\n" diff --git a/erts/emulator/test/map_SUITE_data/badmap_17.beam b/erts/emulator/test/map_SUITE_data/badmap_17.beam Binary files differindex 277fc34b94..6f79bb8c2c 100644 --- a/erts/emulator/test/map_SUITE_data/badmap_17.beam +++ b/erts/emulator/test/map_SUITE_data/badmap_17.beam diff --git a/erts/emulator/test/map_SUITE_data/badmap_17.erl b/erts/emulator/test/map_SUITE_data/badmap_17.erl index 0ec65e0e33..887fc2e5e3 100644 --- a/erts/emulator/test/map_SUITE_data/badmap_17.erl +++ b/erts/emulator/test/map_SUITE_data/badmap_17.erl @@ -1,7 +1,7 @@ -module(badmap_17). -export([update/1]). -%% Compile this source file with OTP 17. +%% Compile this source file with OTP 17.0. update(Map) -> try @@ -17,10 +17,42 @@ update(Map) -> catch error:{badmap,Map} -> ok - end. + end, + try + update_3(Map), + error(update_did_not_fail) + catch + error:{badmap,Map} -> + ok + end, + ok = update_4(Map), + ok = update_5(Map), + ok. update_1(M) -> M#{a=>42}. update_2(M) -> M#{a:=42}. + +update_3(M) -> + id(M), + M#{a=>42}. + +update_4(M) when M#{a=>b} =:= M -> + did_not_fail; +update_4(_) -> + ok. + +update_5(M) -> + id(M), + case id(true) of + true when M#{a=>b} =:= M -> + did_not_fail; + true -> + ok + end. + +id(I) -> + I. + diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 05c250125d..bec2291867 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -25,13 +25,14 @@ %%-define(CHECK(Exp,Got), Exp = Got). -include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). -export([all/0, suite/0, groups/0, init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2, basic/1, reload_error/1, upgrade/1, heap_frag/1, t_on_load/1, - select/1, + select/1, select_steal/1, monitor_process_a/1, monitor_process_b/1, monitor_process_c/1, @@ -42,9 +43,9 @@ types/1, many_args/1, binaries/1, get_string/1, get_atom/1, maps/1, api_macros/1, - from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, + from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, - threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, get_length/1, make_atom/1, make_string/1, reverse_list_test/1, otp_9828/1, @@ -61,7 +62,8 @@ nif_internal_hash_salted/1, nif_phash2/1, nif_whereis/1, nif_whereis_parallel/1, - nif_whereis_threaded/1, nif_whereis_proxy/1 + nif_whereis_threaded/1, nif_whereis_proxy/1, + nif_ioq/1 ]). -export([many_args_100/100]). @@ -78,7 +80,7 @@ all() -> [{group, G} || G <- api_groups()] ++ [reload_error, heap_frag, types, many_args, - select, + select, select_steal, {group, monitor}, monitor_frenzy, hipe, @@ -99,7 +101,8 @@ all() -> nif_internal_hash, nif_internal_hash_salted, nif_phash2, - nif_whereis, nif_whereis_parallel, nif_whereis_threaded]. + nif_whereis, nif_whereis_parallel, nif_whereis_threaded, + nif_ioq]. groups() -> [{G, [], api_repeaters()} || G <- api_groups()] @@ -142,7 +145,8 @@ init_per_testcase(nif_whereis_threaded, Config) -> true -> Config; false -> {skip, "No thread support"} end; -init_per_testcase(select, Config) -> +init_per_testcase(Select, Config) when Select =:= select; + Select =:= select_steal -> case os:type() of {win32,_} -> {skip, "Test not yet implemented for windows"}; @@ -150,6 +154,9 @@ init_per_testcase(select, Config) -> Config end; init_per_testcase(_Case, Config) -> + %% Clear any resource dtor data before test starts in case another tc + %% left it in a bad state + catch last_resource_dtor_call(), Config. end_per_testcase(t_on_load, _Config) -> @@ -588,7 +595,71 @@ select_3(_Config) -> {_,_,2} = last_resource_dtor_call(), ok. -check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok; +%% @doc The stealing child process for the select_steal test. Duplicates given +%% W/RFds and runs select on them to steal +select_steal_child_process(Parent, RFd) -> + %% Duplicate the resource with the same FD + {R2Fd, _R2Ptr} = dupe_resource_nif(RFd), + Ref2 = make_ref(), + + %% Try to select from the child pid (steal from parent) + ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)), + ?assertEqual([], flush(0)), + ?assertEqual(eagain, read_nif(R2Fd, 1)), + + %% Check that now events arrive to this temporary process + Parent ! {self(), stage1}, % signal parent to send the <<"stolen1">> + + %% Receive <<"stolen1">> via enif_select + ?assertEqual(0, select_nif(R2Fd, ?ERL_NIF_SELECT_READ, R2Fd, null, Ref2)), + ?assertMatch([{select, R2Fd, Ref2, ready_input}], flush()), + ?assertEqual(<<"stolen1">>, read_nif(R2Fd, 7)), + + clear_select_nif(R2Fd), + + % do not do this here - stop_selecting(R2Fd, R2Rsrc, Ref2), + Parent ! {self(), done}. + +%% @doc Similar to select/1 test, make a double ended pipe. Then try to steal +%% the socket, see what happens. +select_steal(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + Ref = make_ref(), + {{RFd, RPtr}, {WFd, WPtr}} = pipe_nif(), + + %% Bind the socket to current pid in enif_select + ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, null, Ref)), + ?assertEqual([], flush(0)), + + %% Spawn a process and do some stealing + Parent = self(), + Pid = spawn_link(fun() -> select_steal_child_process(Parent, RFd) end), + + %% Signal from the child to send the first message + {Pid, stage1} = receive_any(), + ?assertEqual(ok, write_nif(WFd, <<"stolen1">>)), + + ?assertMatch([{Pid, done}], flush(1)), % synchronize with the child + + %% Try to select from the parent pid (steal back) + ?assertEqual(0, select_nif(RFd, ?ERL_NIF_SELECT_READ, RFd, Pid, Ref)), + + %% Ensure that no data is hanging and close. + %% Rfd is stolen at this point. + check_stop_ret(select_nif(WFd, ?ERL_NIF_SELECT_STOP, WFd, null, Ref)), + ?assertMatch([{fd_resource_stop, WPtr, _}], flush()), + {1, {WPtr, 1}} = last_fd_stop_call(), + + check_stop_ret(select_nif(RFd, ?ERL_NIF_SELECT_STOP, RFd, null, Ref)), + ?assertMatch([{fd_resource_stop, RPtr, _}], flush()), + {1, {RPtr, 1}} = last_fd_stop_call(), + + ?assert(is_closed_nif(WFd)), + + ok. + +check_stop_ret(?ERL_NIF_SELECT_STOP_CALLED) -> ok; check_stop_ret(?ERL_NIF_SELECT_STOP_SCHEDULED) -> ok. write_full(W, C) -> @@ -1721,14 +1792,9 @@ send2(Config) when is_list(Config) -> %% Send msg from user thread send_threaded(Config) when is_list(Config) -> - case erlang:system_info(smp_support) of - true -> - send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end), - send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end), - ok; - false -> - {skipped,"No threaded send on non-SMP"} - end. + send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end), + send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end), + ok. send2_do1(SendBlobF) -> @@ -2886,11 +2952,15 @@ nif_whereis_parallel(Config) when is_list(Config) -> true = lists:all(PidReg, Procs), %% tell them all to 'fire' as fast as we can - [P ! {Ref, send_proc} || {_, P, _} <- Procs], + repeat(10, fun(_) -> + [P ! {Ref, send_proc} || {_, P, _} <- Procs] + end, void), %% each gets forwarded through two processes - true = lists:all(RecvNum, NSeq), - true = lists:all(RecvNum, NSeq), + repeat(10, fun(_) -> + true = lists:all(RecvNum, NSeq), + true = lists:all(RecvNum, NSeq) + end, void), %% tell them all to 'quit' by name [N ! {Ref, quit} || {N, _, _} <- Procs], @@ -2953,6 +3023,180 @@ nif_whereis_proxy(Ref) -> {Ref, quit} -> ok end. +nif_ioq(Config) -> + ensure_lib_loaded(Config), + + Script = + [{create, a}, + + %% Test enq of erlang term binary + {enqb, a}, + {enqb, a, 3}, + + %% Test enq of non-erlang term binary + {enqbraw,a}, + {enqbraw,a, 5}, + {peek, a}, + {deq, a, 42}, + + %% Test enqv + {enqv, a, 2, 100}, + {deq, a, all}, + + %% This skips all elements but one in the iolist + {enqv, a, 5, iolist_size(nif_ioq_payload(5)) - 1}, + {peek, a}, + + %% Test to enqueue a bunch of refc binaries + {enqv, a, [nif_ioq_payload(refcbin) || _ <- lists:seq(1,20)], 0}, + + %% Enq stuff to destroy with data in queue + {enqv, a, 2, 100}, + {destroy,a}, + + %% Test destroy of new queue + {create, a}, + {destroy,a} + ], + + nif_ioq_run(Script), + + %% Test that only enif_inspect_as_vec works + Payload = nif_ioq_payload(5), + PayloadBin = iolist_to_binary(Payload), + + [begin + PayloadBin = iolist_to_binary(ioq_nif(inspect,Payload,Stack,Env)), + <<>> = iolist_to_binary(ioq_nif(inspect,[],Stack,Env)) + end || Stack <- [no_stack, use_stack], Env <- [use_env, no_env]], + + %% Test error cases + + Q = ioq_nif(create), + + {'EXIT', {badarg, _}} = (catch ioq_nif(deq, Q, 1)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, 1, 1234)), + + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [atom_in_list], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [make_ref()], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [256], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [-1], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [#{}], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [1 bsl 64], 0)), + {'EXIT', {badarg, _}} = (catch ioq_nif(enqv, Q, [{tuple}], 0)), + + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [atom_in_list], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [make_ref()], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [256], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [-1], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [#{}], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [1 bsl 64], no_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, [{tuple}], use_stack)), + {'EXIT', {badarg, _}} = (catch ioq_nif(inspect, <<"binary">>, use_stack)), + + ioq_nif(destroy, Q), + + %% Test that the example in the docs works + ExampleQ = ioq_nif(create), + true = ioq_nif(example, ExampleQ, nif_ioq_payload(5)), + ioq_nif(destroy, ExampleQ), + + ok. + + +nif_ioq_run(Script) -> + nif_ioq_run(Script, #{}). + +nif_ioq_run([{Action, Name}|T], State) + when Action =:= enqb; Action =:= enqbraw -> + nif_ioq_run([{Action, Name, heapbin}|T], State); +nif_ioq_run([{Action, Name, Skip}|T], State) + when Action =:= enqb, is_integer(Skip); + Action =:= enqbraw, is_integer(Skip) -> + nif_ioq_run([{Action, Name, heapbin, Skip}|T], State); +nif_ioq_run([{Action, Name, N}|T], State) + when Action =:= enqv; Action =:= enqb; Action =:= enqbraw -> + nif_ioq_run([{Action, Name, N, 0}|T], State); +nif_ioq_run([{Action, Name, N, Skip}|T], State) + when Action =:= enqv; Action =:= enqb; Action =:= enqbraw -> + + #{ q := IOQ, b := B } = Q = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + %% Sanitize the log output a bit so that it doesn't become too large. + H = {Action, Name, try iolist_size(N) of Sz -> Sz catch _:_ -> N end, Skip}, + ct:log("~p", [H]), + + Data = nif_ioq_payload(N), + ioq_nif(Action, IOQ, Data, Skip), + + <<_:Skip/binary, SkippedData/binary>> = iolist_to_binary(Data), + + true = ioq_nif(size, IOQ) == (iolist_size([B|SkippedData])), + + nif_ioq_run(T, State#{ Name := Q#{ b := [B|SkippedData]}}); +nif_ioq_run([{peek, Name} = H|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + Data = ioq_nif(peek, IOQ, ioq_nif(size, IOQ)), + + true = iolist_to_binary(B) == iolist_to_binary(Data), + nif_ioq_run(T, State); +nif_ioq_run([{deq, Name, all}|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + Size = ioq_nif(size, IOQ), + true = Size == iolist_size(B), + nif_ioq_run([{deq, Name, Size}|T], State); +nif_ioq_run([{deq, Name, N} = H|T], State) -> + #{ q := IOQ, b := B } = Q = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + <<_:N/binary,Remain/binary>> = iolist_to_binary(B), + NewQ = Q#{ b := Remain }, + + Sz = ioq_nif(deq, IOQ, N), + + true = Sz == iolist_size(Remain), + true = ioq_nif(size, IOQ) == iolist_size(Remain), + + nif_ioq_run(T, State#{ Name := NewQ }); +nif_ioq_run([{create, Name} = H|T], State) -> + ct:log("~p", [H]), + nif_ioq_run(T, State#{ Name => #{ q => ioq_nif(create), b => [] } }); +nif_ioq_run([{destroy, Name} = H|T], State) -> + #{ q := IOQ, b := B } = maps:get(Name, State), + true = ioq_nif(size, IOQ) == iolist_size(B), + + ct:log("~p", [H]), + + ioq_nif(destroy, IOQ), + + nif_ioq_run(T, maps:remove(Name, State)); +nif_ioq_run([], State) -> + State. + +nif_ioq_payload(N) when is_integer(N) -> + Tail = if N > 3 -> nif_ioq_payload(N-3); true -> [] end, + Head = element(1, lists:split(N,[nif_ioq_payload(subbin), + nif_ioq_payload(heapbin), + nif_ioq_payload(refcbin) | Tail])), + erlang:iolist_to_iovec(Head); +nif_ioq_payload(subbin) -> + Bin = nif_ioq_payload(refcbin), + Sz = size(Bin) - 1, + <<_:8,SubBin:Sz/binary,_/bits>> = Bin, + SubBin; +nif_ioq_payload(heapbin) -> + <<"a literal heap binary">>; +nif_ioq_payload(refcbin) -> + iolist_to_binary([lists:seq(1,255) || _ <- lists:seq(1,255)]); +nif_ioq_payload(Else) -> + Else. %% The NIFs: lib_version() -> undefined. @@ -3018,16 +3262,22 @@ binary_to_term_nif(_, _, _) -> ?nif_stub. port_command_nif(_, _) -> ?nif_stub. format_term_nif(_,_) -> ?nif_stub. select_nif(_,_,_,_,_) -> ?nif_stub. +dupe_resource_nif(_) -> ?nif_stub. pipe_nif() -> ?nif_stub. write_nif(_,_) -> ?nif_stub. read_nif(_,_) -> ?nif_stub. is_closed_nif(_) -> ?nif_stub. +clear_select_nif(_) -> ?nif_stub. last_fd_stop_call() -> ?nif_stub. alloc_monitor_resource_nif() -> ?nif_stub. monitor_process_nif(_,_,_,_) -> ?nif_stub. demonitor_process_nif(_,_) -> ?nif_stub. compare_monitors_nif(_,_) -> ?nif_stub. monitor_frenzy_nif(_,_,_,_) -> ?nif_stub. +ioq_nif(_) -> ?nif_stub. +ioq_nif(_,_) -> ?nif_stub. +ioq_nif(_,_,_) -> ?nif_stub. +ioq_nif(_,_,_,_) -> ?nif_stub. %% whereis whereis_send(_Type,_Name,_Msg) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 307d1c390f..79560a38aa 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -186,6 +186,12 @@ static ErlNifResourceTypeInit frenzy_rt_init = { static ErlNifResourceType* whereis_resource_type; static void whereis_thread_resource_dtor(ErlNifEnv* env, void* obj); +static ErlNifResourceType* ioq_resource_type; + +static void ioq_resource_dtor(ErlNifEnv* env, void* obj); +struct ioq_resource { + ErlNifIOQueue *q; +}; static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp) { @@ -243,6 +249,10 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) whereis_resource_type = enif_open_resource_type(env, NULL, "nif_SUITE.whereis", whereis_thread_resource_dtor, ERL_NIF_RT_CREATE, NULL); + ioq_resource_type = enif_open_resource_type(env,NULL,"ioq", + ioq_resource_dtor, + ERL_NIF_RT_CREATE, NULL); + atom_false = enif_make_atom(env,"false"); atom_true = enif_make_atom(env,"true"); atom_self = enif_make_atom(env,"self"); @@ -2430,7 +2440,6 @@ static ERL_NIF_TERM format_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg return enif_make_binary(env,&obin); } - static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc) { if (!enif_get_resource(env, term, fd_resource_type, (void**)rsrc)) { @@ -2439,6 +2448,13 @@ static int get_fd(ErlNifEnv* env, ERL_NIF_TERM term, struct fd_resource** rsrc) return 1; } +/* Returns: badarg + * Or an enif_select result, which is a combination of bits: + * ERL_NIF_SELECT_STOP_CALLED = 1 + * ERL_NIF_SELECT_STOP_SCHEDULED = 2 + * ERL_NIF_SELECT_INVALID_EVENT = 4 + * ERL_NIF_SELECT_FAILED = 8 + */ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* fdr; @@ -2470,6 +2486,9 @@ static ERL_NIF_TERM select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv } #ifndef __WIN32__ +/* + * Create a read-write pipe with two fds (to read and to write) + */ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* read_rsrc; @@ -2505,6 +2524,30 @@ static ERL_NIF_TERM pipe_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] enif_make_tuple2(env, write_fd, make_pointer(env, write_rsrc))); } +/* + * Create (dupe) of a resource with the same fd, to test stealing + */ +static ERL_NIF_TERM dupe_resource_nif(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) { + struct fd_resource* orig_rsrc; + + if (!get_fd(env, argv[0], &orig_rsrc)) { + return enif_make_badarg(env); + } else { + struct fd_resource* new_rsrc; + ERL_NIF_TERM new_fd; + + new_rsrc = enif_alloc_resource(fd_resource_type, + sizeof(struct fd_resource)); + new_rsrc->fd = orig_rsrc->fd; + new_rsrc->was_selected = 0; + new_fd = enif_make_resource(env, new_rsrc); + enif_release_resource(new_rsrc); + + return enif_make_tuple2(env, new_fd, make_pointer(env, new_rsrc)); + } +} + static ERL_NIF_TERM write_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { struct fd_resource* fdr; @@ -2580,6 +2623,20 @@ static ERL_NIF_TERM is_closed_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a return fdr->fd < 0 ? atom_true : atom_false; } + +static ERL_NIF_TERM clear_select_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + struct fd_resource* fdr = NULL; + + if (!get_fd(env, argv[0], &fdr)) + return enif_make_badarg(env); + + fdr->fd = -1; + fdr->was_selected = 0; + + return atom_ok; +} + #endif /* !__WIN32__ */ @@ -3158,7 +3215,231 @@ static void frenzy_resource_down(ErlNifEnv* env, void* obj, ErlNifPid* pid, abort(); } +/*********** testing ioq ************/ + +static void ioq_resource_dtor(ErlNifEnv* env, void* obj) { + +} + +#ifndef __WIN32__ +static int writeiovec(ErlNifEnv *env, ERL_NIF_TERM term, ERL_NIF_TERM *tail, ErlNifIOQueue *q, int fd) { + ErlNifIOVec vec, *iovec = &vec; + SysIOVec *sysiovec; + int saved_errno; + int iovcnt, n; + + if (!enif_inspect_iovec(env, 64, term, tail, &iovec)) + return -2; + + if (enif_ioq_size(q) > 0) { + /* If the I/O queue contains data we enqueue the iovec and then + peek the data to write out of the queue. */ + if (!enif_ioq_enqv(q, iovec, 0)) + return -3; + + sysiovec = enif_ioq_peek(q, &iovcnt); + } else { + /* If the I/O queue is empty we skip the trip through it. */ + iovcnt = iovec->iovcnt; + sysiovec = iovec->iov; + } + + /* Attempt to write the data */ + n = writev(fd, sysiovec, iovcnt); + saved_errno = errno; + + if (enif_ioq_size(q) == 0) { + /* If the I/O queue was initially empty we enqueue any + remaining data into the queue for writing later. */ + if (n >= 0 && !enif_ioq_enqv(q, iovec, n)) + return -3; + } else { + /* Dequeue any data that was written from the queue. */ + if (n > 0 && !enif_ioq_deq(q, n, NULL)) + return -4; + } + + /* return n, which is either number of bytes written or -1 if + some error happened */ + errno = saved_errno; + return n; +} +#endif + +static ERL_NIF_TERM ioq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + struct ioq_resource *ioq; + ERL_NIF_TERM ret; + if (enif_is_identical(argv[0], enif_make_atom(env, "create"))) { + ErlNifIOQueue *q = enif_ioq_create(ERL_NIF_IOQ_NORMAL); + ioq = (struct ioq_resource *)enif_alloc_resource(ioq_resource_type, + sizeof(*ioq)); + ioq->q = q; + ret = enif_make_resource(env, ioq); + enif_release_resource(ioq); + return ret; + } else if (enif_is_identical(argv[0], enif_make_atom(env, "inspect"))) { + ErlNifIOVec vec, *iovec = NULL; + int i, iovcnt; + ERL_NIF_TERM *elems, tail, list; + ErlNifEnv *myenv = NULL; + + if (enif_is_identical(argv[2], enif_make_atom(env, "use_stack"))) + iovec = &vec; + if (enif_is_identical(argv[3], enif_make_atom(env, "use_env"))) + myenv = env; + if (!enif_inspect_iovec(myenv, ~(size_t)0, argv[1], &tail, &iovec)) + return enif_make_badarg(env); + + iovcnt = iovec->iovcnt; + elems = enif_alloc(sizeof(ERL_NIF_TERM) * iovcnt); + for (i = 0; i < iovcnt; i++) { + ErlNifBinary bin; + if (!enif_alloc_binary(iovec->iov[i].iov_len, &bin)) { + enif_free_iovec(iovec); + enif_free(elems); + return enif_make_badarg(env); + } + memcpy(bin.data, iovec->iov[i].iov_base, iovec->iov[i].iov_len); + elems[i] = enif_make_binary(env, &bin); + } + + if (!myenv) + enif_free_iovec(iovec); + + list = enif_make_list_from_array(env, elems, iovcnt); + enif_free(elems); + return list; + } else { + unsigned skip; + if (!enif_get_resource(env, argv[1], ioq_resource_type, (void**)&ioq) + || !ioq->q) + return enif_make_badarg(env); + + if (enif_is_identical(argv[0], enif_make_atom(env, "example"))) { +#ifndef __WIN32__ + int fd[2], res = 0, cnt = 0, queue_cnt; + ERL_NIF_TERM tail; + char buff[255]; + pipe(fd); + fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK); + fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK); + + /* Write until the pipe buffer is full, which should result in data + * being queued up. */ + for (res = 0; res >= 0; ) { + cnt += res; + res = writeiovec(env, argv[2], &tail, ioq->q, fd[1]); + } + + /* Flush the queue while reading from the other end of the pipe. */ + tail = enif_make_list(env, 0); + while (enif_ioq_size(ioq->q) > 0) { + res = writeiovec(env, tail, &tail, ioq->q, fd[1]); + + if (res < 0 && errno != EAGAIN) { + break; + } else if (res > 0) { + cnt += res; + } + + for (res = 0; res >= 0; ) { + cnt -= res; + res = read(fd[0], buff, sizeof(buff)); + } + } + + close(fd[0]); + close(fd[1]); + + /* Check that we read as much as we wrote */ + if (cnt == 0 && enif_ioq_size(ioq->q) == 0) + return enif_make_atom(env, "true"); + + return enif_make_int(env, cnt); +#else + return enif_make_atom(env, "true"); +#endif + } + if (enif_is_identical(argv[0], enif_make_atom(env, "destroy"))) { + enif_ioq_destroy(ioq->q); + ioq->q = NULL; + return enif_make_atom(env, "false"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqv"))) { + ErlNifIOVec vec, *iovec = &vec; + ERL_NIF_TERM tail; + + if (!enif_get_uint(env, argv[3], &skip)) + return enif_make_badarg(env); + if (!enif_inspect_iovec(env, ~0ul, argv[2], &tail, &iovec)) + return enif_make_badarg(env); + if (!enif_ioq_enqv(ioq->q, iovec, skip)) + return enif_make_badarg(env); + + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqb"))) { + ErlNifBinary bin; + if (!enif_get_uint(env, argv[3], &skip) || + !enif_inspect_binary(env, argv[2], &bin)) + return enif_make_badarg(env); + + if (!enif_ioq_enq_binary(ioq->q, &bin, skip)) + return enif_make_badarg(env); + + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "enqbraw"))) { + ErlNifBinary bin; + ErlNifBinary localbin; + int i; + if (!enif_get_uint(env, argv[3], &skip) || + !enif_inspect_binary(env, argv[2], &bin) || + !enif_alloc_binary(bin.size, &localbin)) + return enif_make_badarg(env); + + memcpy(localbin.data, bin.data, bin.size); + i = enif_ioq_enq_binary(ioq->q, &localbin, skip); + if (!i) + return enif_make_badarg(env); + else + return enif_make_atom(env, "true"); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "peek"))) { + int iovlen, num, i, off = 0; + SysIOVec *iov = enif_ioq_peek(ioq->q, &iovlen); + ErlNifBinary bin; + + if (!enif_get_int(env, argv[2], &num) || !enif_alloc_binary(num, &bin)) + return enif_make_badarg(env); + + for (i = 0; i < iovlen && num > 0; i++) { + int to_copy = num < iov[i].iov_len ? num : iov[i].iov_len; + memcpy(bin.data + off, iov[i].iov_base, to_copy); + num -= to_copy; + off += to_copy; + } + + return enif_make_binary(env, &bin); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "deq"))) { + int num; + size_t sz; + ErlNifUInt64 sz64; + if (!enif_get_int(env, argv[2], &num)) + return enif_make_badarg(env); + + if (!enif_ioq_deq(ioq->q, num, &sz)) + return enif_make_badarg(env); + + sz64 = sz; + + return enif_make_uint64(env, sz64); + } else if (enif_is_identical(argv[0], enif_make_atom(env, "size"))) { + ErlNifUInt64 size = enif_ioq_size(ioq->q); + return enif_make_uint64(env, size); + } + } + + return enif_make_badarg(env); +} static ErlNifFunc nif_funcs[] = { @@ -3243,8 +3524,10 @@ static ErlNifFunc nif_funcs[] = #ifndef __WIN32__ {"pipe_nif", 0, pipe_nif}, {"write_nif", 2, write_nif}, + {"dupe_resource_nif", 1, dupe_resource_nif}, {"read_nif", 2, read_nif}, {"is_closed_nif", 1, is_closed_nif}, + {"clear_select_nif", 1, clear_select_nif}, #endif {"last_fd_stop_call", 0, last_fd_stop_call}, {"alloc_monitor_resource_nif", 0, alloc_monitor_resource_nif}, @@ -3255,7 +3538,11 @@ static ErlNifFunc nif_funcs[] = {"whereis_send", 3, whereis_send}, {"whereis_term", 2, whereis_term}, {"whereis_thd_lookup", 2, whereis_thd_lookup}, - {"whereis_thd_result", 1, whereis_thd_result} + {"whereis_thd_result", 1, whereis_thd_result}, + {"ioq_nif", 1, ioq}, + {"ioq_nif", 2, ioq}, + {"ioq_nif", 3, ioq}, + {"ioq_nif", 4, ioq} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 8e9e3cb05a..be90f929df 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -405,6 +405,7 @@ node_table_gc(Config) when is_list(Config) -> PreKnown = nodes(known), io:format("PreKnown = ~p~n", [PreKnown]), make_node_garbage(0, 200000, 1000, []), + receive after 1000 -> ok end, %% Wait for thread progress... PostKnown = nodes(known), PostAreas = erlang:system_info(allocated_areas), io:format("PostKnown = ~p~n", [PostKnown]), diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index f512fa3a57..730a17d7e8 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -86,6 +86,7 @@ cd_relative/1, close_deaf_port/1, count_fds/1, + dropped_commands/1, dying_port/1, env/1, eof/1, @@ -158,7 +159,7 @@ suite() -> all() -> [otp_6224, {group, stream}, basic_ping, slow_writes, bad_packet, bad_port_messages, {group, options}, - {group, multiple_packets}, parallell, dying_port, + {group, multiple_packets}, parallell, dying_port, dropped_commands, port_program_with_path, open_input_file_port, open_output_file_port, name1, env, huge_env, bad_env, cd, cd_relative, pipe_limit_env, bad_args, @@ -548,6 +549,47 @@ make_dying_port(Config) when is_list(Config) -> Command = lists:concat([PortTest, " -h0 -d -q"]), open_port({spawn, Command}, [stream]). +%% Test that dropped port_commands work correctly. +%% This used to cause a segfault. +%% +%% This testcase creates a port and then lets many processes +%% do parallel commands to it. After a while it closes the +%% port and we are trying to catch the race when doing a +%% command while the port is closing. +dropped_commands(Config) -> + %% Test with output callback + dropped_commands(Config, false, {self(), {command, "1"}}), + %% Test with outputv callback + dropped_commands(Config, true, {self(), {command, "1"}}). + +dropped_commands(Config, Outputv, Cmd) -> + Path = proplists:get_value(data_dir, Config), + os:putenv("ECHO_DRV_USE_OUTPUTV", atom_to_list(Outputv)), + ok = load_driver(Path, "echo_drv"), + [dropped_commands_test(Cmd) || _ <- lists:seq(1, 100)], + timer:sleep(100), + erl_ddll:unload_driver("echo_drv"), + os:unsetenv("ECHO_DRV_USE_OUTPUTV"), + ok. + +dropped_commands_test(Cmd) -> + spawn_monitor( + fun() -> + Port = erlang:open_port({spawn_driver, "echo_drv"}, + [{parallelism, true}]), + [spawn_link(fun() -> spin(Port, Cmd) end) || _ <- lists:seq(1,8)], + timer:sleep(5), + port_close(Port), + timer:sleep(5), + exit(nok) + end), + receive _M -> timer:sleep(5) end. + +spin(P, Cmd) -> + P ! Cmd, + spin(P, Cmd). + + %% Tests that port program with complete path (but without any %% .exe extension) can be started, even if there is a file with %% the same name but without the extension in the same directory. diff --git a/erts/emulator/test/port_SUITE_data/echo_drv.c b/erts/emulator/test/port_SUITE_data/echo_drv.c index 1d39c6a00c..b4370f6455 100644 --- a/erts/emulator/test/port_SUITE_data/echo_drv.c +++ b/erts/emulator/test/port_SUITE_data/echo_drv.c @@ -18,8 +18,11 @@ typedef struct _erl_drv_data EchoDrvData; static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command); static void echo_drv_stop(EchoDrvData *data_p); -static void echo_drv_output(ErlDrvData drv_data, char *buf, - ErlDrvSizeT len); +static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len); +static ErlDrvSSizeT echo_control(ErlDrvData drv_data, + unsigned int command, char *buf, + ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen); +static void echo_outputv(ErlDrvData drv_data, ErlIOVec *ev); static void echo_drv_finish(void); static ErlDrvEntry echo_drv_entry = { @@ -32,9 +35,9 @@ static ErlDrvEntry echo_drv_entry = { "echo_drv", echo_drv_finish, NULL, /* handle */ - NULL, /* control */ + echo_control, /* control */ NULL, /* timeout */ - NULL, /* outputv */ + echo_outputv, /* outputv */ NULL, /* ready_async */ NULL, NULL, @@ -56,6 +59,14 @@ static ErlDrvEntry echo_drv_entry = { DRIVER_INIT(echo_drv) { + char buf[10]; + size_t bufsz = sizeof(buf); + char *use_outputv; + use_outputv = (erl_drv_getenv("ECHO_DRV_USE_OUTPUTV", buf, &bufsz) == 0 + ? buf + : "false"); + if (strcmp(use_outputv, "true") != 0) + echo_drv_entry.outputv = NULL; return &echo_drv_entry; } @@ -87,3 +98,15 @@ static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) { static void echo_drv_finish() { } + +static ErlDrvSSizeT echo_control(ErlDrvData drv_data, + unsigned int command, char *buf, + ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) +{ + return 0; +} + +static void echo_outputv(ErlDrvData drv_data, ErlIOVec *ev) +{ + return; +} diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl index c78dc754a9..a1986397a8 100644 --- a/erts/emulator/test/port_trace_SUITE.erl +++ b/erts/emulator/test/port_trace_SUITE.erl @@ -78,13 +78,6 @@ end_per_group(_GroupName, Config) -> Config. -init_per_testcase(driver_remote_send_term, Config) -> - case erlang:system_info(smp_support) of - false -> - {skip,"Only supported on smp systems"}; - true -> - init_per_testcase(driver_remote_send_term_smp, Config) - end; init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> erlang:trace(all, false, [all]), os:unsetenv("OUTPUTV"), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 6ded7ff1c9..a9f20f9928 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -152,7 +152,11 @@ spawn_with_binaries(Config) when is_list(Config) -> TwoMeg = lists:duplicate(1024, L), Fun = fun() -> spawn(?MODULE, binary_owner, [list_to_binary(TwoMeg)]), receive after 1 -> ok end end, - test_server:do_times(150, Fun), + Iter = case test_server:is_valgrind() of + true -> 10; + false -> 150 + end, + test_server:do_times(Iter, Fun), ok. binary_owner(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl index 43ae749498..49da94a775 100644 --- a/erts/emulator/test/register_SUITE.erl +++ b/erts/emulator/test/register_SUITE.erl @@ -44,14 +44,7 @@ all() -> -define(OTP_8099_NAME, otp_8099_reg_proc). otp_8099(Config) when is_list(Config) -> - case catch erlang:system_info(lock_counting) of - true -> {skipped, - "Lock counting enabled. Current lock counting " - "implementation cannot handle this many " - "processes."}; - _ -> - otp_8099_test(1000000) - end. + otp_8099_test(1000000). otp_8099_test(0) -> ok; diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index af33de237c..7afb82a1b8 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -57,6 +57,7 @@ scheduler_suspend_basic/1, scheduler_suspend/1, dirty_scheduler_threads/1, + poll_threads/1, reader_groups/1]). suite() -> @@ -72,6 +73,7 @@ all() -> {group, scheduler_bind}, scheduler_threads, scheduler_suspend_basic, scheduler_suspend, dirty_scheduler_threads, + poll_threads, reader_groups]. groups() -> @@ -1083,7 +1085,6 @@ sbt_test(Config, CpuTCmd, ClBt, Bt, LP) -> ok. scheduler_threads(Config) when is_list(Config) -> - SmpSupport = erlang:system_info(smp_support), {Sched, SchedOnln, _} = get_sstate(Config, ""), %% Configure half the number of both the scheduler threads and %% the scheduler threads online. @@ -1095,10 +1096,7 @@ scheduler_threads(Config) when is_list(Config) -> %% setting using +SP to 50% scheduler threads and 25% scheduler %% threads online. The result should be 2x scheduler threads and %% 1x scheduler threads online. - TwiceSched = case SmpSupport of - false -> 1; - true -> Sched*2 - end, + TwiceSched = Sched*2, FourSched = integer_to_list(Sched*4), FourSchedOnln = integer_to_list(SchedOnln*4), CombinedCmd1 = "+S "++FourSched++":"++FourSchedOnln++" +SP50:25", @@ -1121,8 +1119,8 @@ scheduler_threads(Config) when is_list(Config) -> ResetCmd = "+S "++FourSched++":"++FourSchedOnln++" +S 0:0", {LProc, LProcAvail, _} = get_sstate(Config, ResetCmd), %% Test negative +S settings, but only for SMP-enabled emulators - case {SmpSupport, LProc > 1, LProcAvail > 1} of - {true, true, true} -> + case {LProc > 1, LProcAvail > 1} of + {true, true} -> SchedMinus1 = LProc-1, SchedOnlnMinus1 = LProcAvail-1, {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1"), @@ -1157,9 +1155,6 @@ dirty_scheduler_threads_test(Config) -> ok. dirty_schedulers_online_test() -> - dirty_schedulers_online_test(erlang:system_info(smp_support)). -dirty_schedulers_online_test(false) -> ok; -dirty_schedulers_online_test(true) -> dirty_schedulers_online_smp_test(erlang:system_info(schedulers_online)). dirty_schedulers_online_smp_test(SchedOnln) when SchedOnln < 4 -> ok; dirty_schedulers_online_smp_test(SchedOnln) -> @@ -1453,6 +1448,79 @@ sst5_loop(N) -> erlang:system_flag(multi_scheduling, unblock_normal), sst5_loop(N-1). +poll_threads(Config) when is_list(Config) -> + {Conc, PollType, KP} = get_ioconfig(Config), + {Sched, SchedOnln, _} = get_sstate(Config, ""), + + [1, 1] = get_ionum(Config,"+IOt 2 +IOp 2"), + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 5"), + + [1, 1] = get_ionum(Config, "+S 2 +IOPt 100 +IOPp 100"), + + if + Conc -> + [5] = get_ionum(Config,"+IOt 5 +IOp 1"), + [3, 2] = get_ionum(Config,"+IOt 5 +IOp 2"), + [2, 2, 2, 2, 2] = get_ionum(Config,"+IOt 10 +IOPp 50"), + + [2] = get_ionum(Config, "+S 2 +IOPt 100"), + [4] = get_ionum(Config, "+S 4 +IOPt 100"), + [4] = get_ionum(Config, "+S 4:2 +IOPt 100"), + [4, 4] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"), + + fail = get_ionum(Config, "+IOt 1 +IOp 2"), + + ok; + not Conc -> + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 1"), + [1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 5 +IOp 2"), + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config,"+IOt 10 +IOPp 50"), + + [1, 1] = get_ionum(Config, "+S 2 +IOPt 100"), + [1, 1, 1, 1] = get_ionum(Config, "+S 4 +IOPt 100"), + [1, 1, 1, 1] = get_ionum(Config, "+S 4:2 +IOPt 100"), + [1, 1, 1, 1, 1, 1, 1, 1] = get_ionum(Config, "+S 8 +IOPt 100 +IOPp 25"), + + [1] = get_ionum(Config, "+IOt 1 +IOp 2"), + + ok + end, + + fail = get_ionum(Config, "+IOt 1 +IOPp 101"), + fail = get_ionum(Config, "+IOt 0"), + fail = get_ionum(Config, "+IOPt 101"), + + ok. + +get_ioconfig(Config) -> + [PS | _] = get_iostate(Config, ""), + {proplists:get_value(concurrent_updates, PS), + proplists:get_value(primary, PS), + proplists:get_value(kernel_poll, PS)}. + +get_ionum(Config, Cmd) -> + case get_iostate(Config, Cmd) of + fail -> fail; + PSs -> + lists:reverse( + lists:sort( + [proplists:get_value(poll_threads, PS) || PS <- PSs])) + end. + +get_iostate(Config, Cmd)-> + case start_node(Config, Cmd) of + {ok, Node} -> + [IOStates] = mcall(Node,[fun () -> + erlang:system_info(check_io) + end]), + IO = [IOState || IOState <- IOStates, + proplists:get_value(fallback, IOState) == false], + stop_node(Node), + IO; + {error,timeout} -> + fail + end. + reader_groups(Config) when is_list(Config) -> %% White box testing. These results are correct, but other results %% could be too... @@ -1777,18 +1845,24 @@ mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> Ref = make_ref(), - spawn_link(Node, - fun () -> - Res = Fun(), - unlink(Parent), - Parent ! {Ref, Res} - end), - Ref + Pid = spawn(Node, + fun () -> + Res = Fun(), + unlink(Parent), + Parent ! {Ref, Res} + end), + MRef = erlang:monitor(process, Pid), + {Ref, MRef} end, Funs), - lists:map(fun (Ref) -> + lists:map(fun ({Ref, MRef}) -> receive {Ref, Res} -> - Res + receive + {'DOWN',MRef,_,_,_} -> + Res + end; + {'DOWN',MRef,_,_,Reason} -> + Reason end end, Refs). diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl index f1d11d1814..61a8617165 100644 --- a/erts/emulator/test/signal_SUITE.erl +++ b/erts/emulator/test/signal_SUITE.erl @@ -139,71 +139,66 @@ pending_exit_gc(Config) when is_list(Config) -> pending_exit_test(self(), gc). pending_exit_test(From, Type) -> - case catch erlang:system_info(smp_support) of - true -> - OTE = process_flag(trap_exit, true), - Ref = make_ref(), - Master = self(), - ExitBySignal = case Type of - gc -> - lists:duplicate(10000, - exit_by_signal); - _ -> - exit_by_signal - end, - Pid = spawn_link( - fun () -> - receive go -> ok end, - false = have_pending_exit(), - exit = fake_exit(From, - self(), - ExitBySignal), - true = have_pending_exit(), - Master ! {self(), Ref, Type}, - case Type of - gc -> - force_gc(), - erlang:yield(); - unlink -> - unlink(From); - trap_exit -> - process_flag(trap_exit, true); - 'receive' -> - receive _ -> ok - after 0 -> ok - end; - exit -> - ok - end, - exit(exit_by_myself) - end), - Mon = erlang:monitor(process, Pid), - Pid ! go, - Reason = receive - {'DOWN', Mon, process, Pid, R} -> - receive - {Pid, Ref, Type} -> - ok - after 0 -> - ct:fail(premature_exit) - end, - case Type of - exit -> - exit_by_myself = R; - _ -> - ExitBySignal = R - end + OTE = process_flag(trap_exit, true), + Ref = make_ref(), + Master = self(), + ExitBySignal = case Type of + gc -> + lists:duplicate(10000, + exit_by_signal); + _ -> + exit_by_signal + end, + Pid = spawn_link( + fun () -> + receive go -> ok end, + false = have_pending_exit(), + exit = fake_exit(From, + self(), + ExitBySignal), + true = have_pending_exit(), + Master ! {self(), Ref, Type}, + case Type of + gc -> + force_gc(), + erlang:yield(); + unlink -> + unlink(From); + trap_exit -> + process_flag(trap_exit, true); + 'receive' -> + receive _ -> ok + after 0 -> ok + end; + exit -> + ok + end, + exit(exit_by_myself) + end), + Mon = erlang:monitor(process, Pid), + Pid ! go, + Reason = receive + {'DOWN', Mon, process, Pid, R} -> + receive + {Pid, Ref, Type} -> + ok + after 0 -> + ct:fail(premature_exit) end, - receive - {'EXIT', Pid, R2} -> - Reason = R2 - end, - process_flag(trap_exit, OTE), - ok, - {comment, "Test only valid with current SMP emulator."}; - _ -> - {skipped, "SMP support not enabled. Test only valid with current SMP emulator."} - end. + case Type of + exit -> + exit_by_myself = R; + _ -> + ExitBySignal = R + end + end, + receive + {'EXIT', Pid, R2} -> + Reason = R2 + end, + process_flag(trap_exit, OTE), + ok, + {comment, "Test only valid with current SMP emulator."}. diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl index 41bb07b84c..adc6f56c06 100644 --- a/erts/emulator/test/smoke_test_SUITE.erl +++ b/erts/emulator/test/smoke_test_SUITE.erl @@ -88,11 +88,9 @@ native_atomics(Config) when is_list(Config) -> {value,{NA32Key, NA32, _}} = lists:keysearch(NA32Key, 1, EthreadInfo), {value,{NA64Key, NA64, _}} = lists:keysearch(NA64Key, 1, EthreadInfo), {value,{DWNAKey, DWNA, _}} = lists:keysearch(DWNAKey, 1, EthreadInfo), - case {erlang:system_info(build_type), erlang:system_info(smp_support), NA32, NA64, DWNA} of - {opt, true, "no", "no", _} -> + case {erlang:system_info(build_type), NA32, NA64, DWNA} of + {opt, "no", "no", _} -> ct:fail(optimized_smp_runtime_without_native_atomics); - {_, false, "no", "no", _} -> - {comment, "No native atomics"}; _ -> {comment, NA32 ++ " 32-bit, " diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 7690557fda..029a6de897 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -23,8 +23,10 @@ %% Tests the statistics/1 bif. -export([all/0, suite/0, groups/0, + wall_clock_sanity/1, wall_clock_zero_diff/1, wall_clock_update/1, - runtime_zero_diff/1, + runtime_sanity/1, + runtime_zero_diff/1, runtime_update/1, runtime_diff/1, run_queue_one/1, scheduler_wall_time/1, @@ -54,11 +56,23 @@ all() -> groups() -> [{wall_clock, [], - [wall_clock_zero_diff, wall_clock_update]}, + [wall_clock_sanity, wall_clock_zero_diff, wall_clock_update]}, {runtime, [], - [runtime_zero_diff, runtime_update, runtime_diff]}, + [runtime_sanity, runtime_zero_diff, runtime_update, runtime_diff]}, {run_queue, [], [run_queue_one]}]. +wall_clock_sanity(Config) when is_list(Config) -> + erlang:yield(), + {WallClock, _} = statistics(wall_clock), + MT = erlang:monotonic_time(), + Time = erlang:convert_time_unit(MT - erlang:system_info(start_time), + native, millisecond), + io:format("Time=~p WallClock=~p~n", + [Time, WallClock]), + true = WallClock =< Time, + true = Time - 100 =< WallClock, + ok. + %%% Testing statistics(wall_clock). %% Tests that the 'Wall clock since last call' element of the result @@ -102,6 +116,20 @@ wall_clock_update1(0) -> %%% Test statistics(runtime). +runtime_sanity(Config) when is_list(Config) -> + case erlang:system_info(logical_processors_available) of + unknown -> + {skipped, "Don't know available logical processors"}; + LP when is_integer(LP) -> + erlang:yield(), + {RunTime, _} = statistics(runtime), + MT = erlang:monotonic_time(), + Time = erlang:convert_time_unit(MT - erlang:system_info(start_time), + native, millisecond), + io:format("Time=~p RunTime=~p~n", + [Time, RunTime]), + true = RunTime =< Time*LP + end. %% Tests that the difference between the times returned from two consectuitive %% calls to statistics(runtime) is zero. @@ -610,9 +638,7 @@ msacc(Config) -> (aux, 0) -> %% aux will be zero if we do not have smp support %% or no async threads - case erlang:system_info(smp_support) orelse - erlang:system_info(thread_pool_size) > 0 - of + case erlang:system_info(thread_pool_size) > 0 of false -> ok; true -> @@ -648,6 +674,16 @@ msacc_test(TmpFile) -> ets:insert(Tid, {1, hello}), ets:delete(Tid), + %% Check some IO + {ok, L} = gen_tcp:listen(0, [{active, true},{reuseaddr,true}]), + {ok, Port} = inet:port(L), + Pid = spawn(fun() -> + {ok, S} = gen_tcp:accept(L), + (fun F() -> receive M -> F() end end)() + end), + {ok, C} = gen_tcp:connect("localhost", Port, []), + [begin gen_tcp:send(C,"hello"),timer:sleep(1) end || _ <- lists:seq(1,100)], + %% Collect some garbage [erlang:garbage_collect() || _ <- lists:seq(1,100)], diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl index 9b678fcff9..c9be54f668 100644 --- a/erts/emulator/test/system_profile_SUITE.erl +++ b/erts/emulator/test/system_profile_SUITE.erl @@ -146,9 +146,8 @@ do_runnable_ports({TsType, TsTypeFlag}, Config) -> %% Tests system_profiling with scheduler. scheduler(Config) when is_list(Config) -> - case {erlang:system_info(smp_support), erlang:system_info(schedulers_online)} of - {false,_} -> {skipped, "No need for scheduler test when smp support is disabled."}; - {_, 1} -> {skipped, "No need for scheduler test when only one scheduler online."}; + case erlang:system_info(schedulers_online) of + 1 -> {skipped, "No need for scheduler test when only one scheduler online."}; _ -> Nodes = 10, lists:foreach(fun (TsType) -> diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl index 79b681b4d1..baf41180e0 100644 --- a/erts/emulator/test/tuple_SUITE.erl +++ b/erts/emulator/test/tuple_SUITE.erl @@ -134,6 +134,13 @@ t_element(Config) when is_list(Config) -> {'EXIT', {badarg, _}} = (catch element(1, id(42))), {'EXIT', {badarg, _}} = (catch element(id(1.5), id({a,b}))), + %% Make sure that the loader does not reject the module when + %% huge literal index values are used. + {'EXIT', {badarg, _}} = (catch element((1 bsl 24)-1, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 24, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 32, id({a,b,c}))), + {'EXIT', {badarg, _}} = (catch element(1 bsl 64, id({a,b,c}))), + ok. get_elements([Element|Rest], Tuple, Pos) -> diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index feea7432a9..ac3df8bfbf 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -330,7 +330,7 @@ display_check_io(ChkIo) -> ok. get_check_io_info() -> - ChkIo = erlang:system_info(check_io), + ChkIo = driver_SUITE:get_check_io_total(erlang:system_info(check_io)), PendUpdNo = case lists:keysearch(pending_updates, 1, ChkIo) of {value, {pending_updates, PendNo}} -> PendNo; diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 0a30553f71..64a9a49ac8 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -20,13 +20,16 @@ # use strict; use vars qw($BEAM_FORMAT_NUMBER); +use constant COLD => 0; +use constant WARM => 1; +use constant HOT => 2; $BEAM_FORMAT_NUMBER = undef; my $target = \&emulator_output; my $outdir = "."; # Directory for output files. my $verbose = 0; -my $hot = 1; +my $hotness = 1; my $num_file_opcodes = 0; my $wordsize = 32; my %defs; # Defines (from command line). @@ -54,11 +57,6 @@ $pack_mask[4] = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize 'BEAM_LOOSE_MASK', $WHOLE_WORD]; -# Mapping from packagable arguments to number of packed arguments per -# word. Initialized after the wordsize is known. - -my @args_per_word; - # There are two types of instructions: generic and specific. # The generic instructions are those generated by the Beam compiler. # Corresponding to each generic instruction, there is generally a @@ -83,6 +81,10 @@ my %num_specific; my %gen_to_spec; my %specific_op; +# Information about each specific operator. Key is the print name (e.g. get_list_xxy). +# Value is a hash. +my %spec_op_info; + my %gen_arity; my @gen_arity; @@ -91,17 +93,22 @@ my @op_to_name; my @obsolete; -my %macro; -my %macro_flags; +# Instructions and micro instructions implemented in C. +my %c_code; # C code block, location, arguments. +my %c_code_used; # Used or not. + +# Definitions for instructions combined from micro instructions. +my %combined_instrs; -my %hot_code; -my %cold_code; +my @generated_code; # Generated code. +my %sort_order; my @unnumbered_generic; my %unnumbered; my %is_transformed; + # # Pre-processor. # @@ -138,13 +145,15 @@ my %arg_size = ('r' => 0, # x(0) - x register zero 'n' => 0, # NIL (implicit) 'c' => 1, # tagged constant (integer, atom, nil) 's' => 1, # tagged source; any of the above + 'S' => 1, # tagged source register (x or y) 'd' => 1, # tagged destination register (r, x, y) 'f' => 1, # failure label 'j' => 1, # either 'f' or 'p' 'e' => 1, # pointer to export entry 'L' => 0, # label - 'I' => 1, # untagged integer - 't' => 1, # untagged integer -- can be packed + 't' => 1, # untagged integer (12 bits) -- can be packed + 'I' => 1, # untagged integer (32 bits) -- can be packed + 'W' => 1, # untagged integer/pointer (one word) 'b' => 1, # pointer to bif 'A' => 1, # arity value 'P' => 1, # byte offset into tuple or stack @@ -186,16 +195,16 @@ sub define_type_bit { define_type_bit('s', $type_bit{'d'} | $type_bit{'i'} | $type_bit{'a'} | $type_bit{'n'} | $type_bit{'q'}); + define_type_bit('S', $type_bit{'d'}); define_type_bit('j', $type_bit{'f'} | $type_bit{'p'}); # Aliases (for matching purposes). - define_type_bit('I', $type_bit{'u'}); define_type_bit('t', $type_bit{'u'}); + define_type_bit('I', $type_bit{'u'}); + define_type_bit('W', $type_bit{'u'}); define_type_bit('A', $type_bit{'u'}); define_type_bit('L', $type_bit{'u'}); define_type_bit('b', $type_bit{'u'}); - define_type_bit('N', $type_bit{'u'}); - define_type_bit('U', $type_bit{'u'}); define_type_bit('e', $type_bit{'u'}); define_type_bit('P', $type_bit{'u'}); define_type_bit('Q', $type_bit{'u'}); @@ -222,6 +231,12 @@ $match_engine_ops{'TOP_fail'} = 1; sanity("tag '$tag': primitive tags must be named with lowercase letters") unless $tag =~ /^[a-z]$/; } + + foreach my $tag (keys %arg_size) { + defined $type_bit{$tag} or + sanity("the tag '$tag' has a size in %arg_size, " . + "but has no defined bit pattern"); + } } # @@ -240,27 +255,56 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) { die "$0: Bad option: -$_\n"; } +if ($wordsize == 32) { + $defs{'ARCH_32'} = 1; + $defs{'ARCH_64'} = 0; +} elsif ($wordsize == 64) { + $defs{'ARCH_32'} = 0; + $defs{'ARCH_64'} = 1; +} + # # Initialize number of arguments per packed word. # -$args_per_word[2] = 2; -$args_per_word[3] = 3; -$args_per_word[4] = 2; -$args_per_word[5] = 3; -$args_per_word[6] = 3; - if ($wordsize == 64) { $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', $WHOLE_WORD]; - $args_per_word[4] = 4; +} + +# +# Add placeholders for built-in macros. +# + +$c_code{'IS_PACKED'} = ['$Expr',"built-in macro",('Expr')]; +$c_code{'ARG_POSITION'} = ['$Expr',"built-in macro",('Expr')]; +foreach my $name (keys %c_code) { + $c_code_used{$name} = 1; } # # Parse the input files. # +my $in_c_code = ''; +my $c_code_block; +my $c_code_loc; +my @c_args; + while (<>) { my($op_num); + if ($in_c_code) { + if (/^\}/) { + my $name = $in_c_code; + my $block = $c_code_block; + $in_c_code = ''; + $block =~ s/^ //mg; + chomp $block; + $c_code{$name} = [$block,$c_code_loc,@c_args]; + } else { + $c_code_block .= $_; + } + next; + } chomp; if (s/\\$//) { $_ .= <>; @@ -268,6 +312,7 @@ while (<>) { } next if /^\s*$/; next if /^\#/; + next if m@^//@; # # Handle %if. @@ -316,30 +361,16 @@ while (<>) { } # - # Handle %hot/%cold. + # Handle %hot, %warm, and %cold. # if (/^\%hot/) { - $hot = 1; + $hotness = HOT; next; + } elsif (/^\%warm/) { + $hotness = WARM; + next; } elsif (/^\%cold/) { - $hot = 0; - next; - } - - # - # Handle macro definitions. - # - if (/^\%macro:(.*)/) { - my($op, $macro, @flags) = split(' ', $1); - defined($macro) and $macro =~ /^-/ and - error("A macro must not start with a hyphen"); - foreach (@flags) { - /^-/ or error("Flags for macros should start with a hyphen"); - } - error("Macro for '$op' is already defined") - if defined $macro{$op}; - $macro{$op} = $macro; - $macro_flags{$op} = join('', @flags); + $hotness = COLD; next; } @@ -352,6 +383,31 @@ while (<>) { } # + # Handle C code blocks. + # + if (/^(\w[\w.]*)\(([^\)]*)\)\s*{/) { + my $name = $1; + $in_c_code = $name; + $c_code_block = ''; + @c_args = parse_c_args($2); + $c_code_loc = "$ARGV($.)"; + if (defined $c_code{$name}) { + my $where = $c_code{$name}->[1]; + error("$name: already defined at $where"); + } + next; + } + + # + # Handle definition of instructions in terms of + # micro instructions. + # + if (/^(\w+)\s*:=\s*([\w.]+)\s*;\s*$/) { + $combined_instrs{$1} = ["$ARGV($.)",$2]; + next; + } + + # # Parse off the number of the operation. # $op_num = undef; @@ -402,7 +458,7 @@ while (<>) { if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) { error("specific instructions may not be specified for obsolete instructions"); } - save_specific_ops($name, $arity, $hot, @args); + save_specific_ops($name, $arity, $hotness, @args); if (defined $op_num) { error("specific instructions must not be numbered"); } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) { @@ -449,6 +505,18 @@ $num_file_opcodes = @gen_opname; &$target(); # +# Ensure that all C code implementations have been used. +# +{ + my(@unused) = grep(!$c_code_used{$_}, keys %c_code); + foreach my $unused (@unused) { + my(undef,$where) = @{$c_code{$unused}}; + warn "$where: $unused is unused\n"; + } + die "\n" if @unused; +} + +# # Produce output needed by the emulator/loader. # @@ -458,6 +526,37 @@ sub emulator_output { my $key; # Loop variable. # + # Generate code and meta information for all instructions. + # + foreach $key (keys %specific_op) { + foreach (@{$specific_op{$key}}) { + my($name, $hotness, @args) = @$_; + my $sign = join('', @args); + my $print_name = print_name($name, @args); + + my($size, $code, $pack_spec) = cg_basic($name, @args); + if (defined $code) { + $code = "OpCase($print_name):\n$code"; + push @generated_code, [$hotness,$code,($print_name)]; + } + + # Note: Some of the information below will be modified + # for combined instructions. + my %info = ('size' => $size, + 'pack_spec' => $pack_spec, + 'adj' => 0, + 'args' => \@args); + $spec_op_info{$print_name} = \%info; + } + } + + # + # Combine micro instruction into instruction blocks and generate + # code for them. + # + combine_micro_instructions(); + + # # Information about opcodes (beam_opcodes.c). # $name = "$outdir/beam_opcodes.c"; @@ -488,7 +587,7 @@ sub emulator_output { # # Generate code for specific ops. # - my($spec_opnum) = 0; + my $spec_opnum = 0; print "const OpEntry opc[] = {\n"; foreach $key (sort keys %specific_op) { $gen_to_spec{$key} = $spec_opnum; @@ -506,37 +605,21 @@ sub emulator_output { # The primitive types should sort before other types. - my($sort_key) = $sign; + my $sort_key = $sign; eval "\$sort_key =~ tr/$genop_types/./"; $sort_key .= ":$sign"; - $items{$sort_key} = [$name, $hot, $sign, @args]; + my $print_name = print_name($name, @args); + $items{$sort_key} = $print_name; } # # Now call the generator for the sorted result. # - foreach (sort keys %items) { - my($name, $hot, $sign, @args) = @{$items{$_}}; + foreach my $sort_key (sort keys %items) { + my $print_name = $items{$sort_key}; + my $info = $spec_op_info{$print_name}; + my(@args) = @{$info->{'args'}}; my $arity = @args; - my($instr) = "${name}_$sign"; - $instr =~ s/_$//; - - # - # Call a generator to calculate size and generate macros - # for the emulator. - # - my($size, $code, $pack) = basic_generator($name, $hot, @args); - - # - # Save the generated $code for later. - # - if (defined $code) { - if ($hot) { - push(@{$hot_code{$code}}, $instr); - } else { - push(@{$cold_code{$code}}, $instr); - } - } # # Calculate the bit mask which should be used to match this @@ -558,7 +641,6 @@ sub emulator_output { } printf "/* %3d */ ", $spec_opnum; - my $print_name = $sign ne '' ? "${name}_$sign" : $name; my $init = "{"; my $sep = ""; foreach (@bits) { @@ -566,8 +648,12 @@ sub emulator_output { $sep = ","; } $init .= "}"; - init_item($print_name, $init, $involves_r, $size, $pack, $sign); - $op_to_name[$spec_opnum] = $instr; + my $adj = $info->{'adj'}; + my $size = $info->{'size'}; + my $pack_spec = $info->{'pack_spec'}; + my $sign = join '', @args; + init_item($print_name, $init, $involves_r, $size, $adj, $pack_spec, $sign); + $op_to_name[$spec_opnum] = $print_name; $spec_opnum++; } } @@ -646,9 +732,9 @@ sub emulator_output { print "#if !defined(ARCH_64)\n"; print qq[ #error "64-bit architecture assumed, but ARCH_64 not defined"\n]; print "#endif\n"; - print "#define BEAM_WIDE_MASK 0xFFFFUL\n"; - print "#define BEAM_LOOSE_MASK 0xFFFFUL\n"; - print "#define BEAM_TIGHT_MASK 0xFFFFUL\n"; + print "#define BEAM_WIDE_MASK 0xFFFFFFFFull\n"; + print "#define BEAM_LOOSE_MASK 0xFFFFull\n"; + print "#define BEAM_TIGHT_MASK 0xFFFFull\n"; print "#define BEAM_WIDE_SHIFT 32\n"; print "#define BEAM_LOOSE_SHIFT 16\n"; print "#define BEAM_TIGHT_SHIFT 16\n"; @@ -750,13 +836,23 @@ sub emulator_output { $name = "$outdir/beam_hot.h"; open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; comment('C'); - print_code(\%hot_code); + print_code(HOT); + + $name = "$outdir/beam_warm.h"; + open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; + comment('C'); + print_code(WARM); $name = "$outdir/beam_cold.h"; open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n"; comment('C'); - print_code(\%cold_code); + print_code(COLD); +} +sub print_name { + my($name,@args) = @_; + my $sign = join '', @args; + $sign ne '' ? "${name}_$sign" : $name; } sub init_item { @@ -784,29 +880,47 @@ sub q { } sub print_code { - my($ref) = @_; - my(%sorted); - my($key, $label); # Loop variables. - - foreach $key (keys %$ref) { - my($sort_key); - my($code) = ''; - foreach $label (@{$ref->{$key}}) { - $code .= "OpCase($label):\n"; - $sort_key = $label; - } - foreach (split("\n", $key)) { - $code .= " $_\n"; - } - $code .= "\n"; - $sorted{$sort_key} = $code; + my($include_hot) = @_; + my %sorted; + + foreach my $ref (@generated_code) { + my($hot,$code,@labels) = @$ref; + next unless $hot == $include_hot; + my($sort_key) = @labels; # Use the first label as sort key. + $sorted{$sort_key} = $code; } foreach (sort keys %sorted) { - print $sorted{$_}; + print_indented_code($sorted{$_}); + } +} + +sub print_indented_code { + my(@code) = @_; + + foreach my $chunk (@code) { + my $indent = 0; + foreach (split "\n", $chunk) { + s/^\s*//; + if (/\}/) { + $indent -= 2; + } + if ($_ eq '') { + print "\n"; + } elsif (/^#/) { + print $_, "\n"; + } else { + print ' ' x $indent, $_, "\n"; + } + if (/\{/) { + $indent += 2; + } + } + print "\n"; } } + # # Produce output needed by the compiler back-end (assembler). # @@ -893,6 +1007,18 @@ sub save_specific_ops { } } +sub parse_c_args { + local($_) = @_; + my @res; + + while (s/^(\w[\w\d]*)\s*//) { + push @res, $1; + s/^,\s*// or last; + } + $_ eq '' or error("garbage in argument list: $_"); + @res; +} + sub error { my(@message) = @_; my($where) = $. ? "$ARGV($.): " : ""; @@ -934,58 +1060,274 @@ sub comment { } # -# Basic implementation of instruction in emulator loop -# (assuming no packing). +# Combine micro instruction into instruction blocks. # +sub combine_micro_instructions { + my %groups; + + # Sanity check, normalize micro instructions. + foreach my $instr (keys %combined_instrs) { + my $ref = $combined_instrs{$instr}; + my($def_loc,$def) = @$ref; + my($group,@subs) = split /[.]/, $def; + my $arity = 0; + @subs = map { "$group.$_" } @subs; + foreach my $s (@subs) { + my $code = $c_code{$s}; + defined $code or + error("$def_loc: no definition of $s"); + $c_code_used{$s} = 1; + my(undef,undef,@c_args) = @{$code}; + $arity += scalar(@c_args); + } + push @{$groups{$group}}, [$instr,$arity,@subs]; + } -sub basic_generator { - my($name, $hot, @args) = @_; - my($size) = 0; - my($macro) = ''; - my($flags) = ''; - my(@f); - my(@f_types); - my($fail_type); - my($prefix) = ''; - my($tmp_arg_num) = 1; - my($pack_spec) = ''; - my($var_decls) = ''; - my($i); - my($no_prefetch) = 0; + # Now generate code for each group. + foreach my $group (sort keys %groups) { + my($hotness,$code,@labels) = + combine_instruction_group($group, @{$groups{$group}}); + push @generated_code, [$hotness,$code,@labels]; + } +} + +sub combine_instruction_group { + my($group,@in_instrs) = @_; + my $gcode = ''; # Code for the entire group. + my $group_hotness = COLD; + + # Get code for the head of the group (if any). + my $head_name = "$group.head"; + $c_code_used{$head_name} = 1; + my $head_code_ref = $c_code{$head_name}; + if (defined $head_code_ref) { + my($head_code,$where,@c_args) = @{$head_code_ref}; + @c_args and error("$where: no arguments allowed for " . + "head function '$head_name()'"); + $gcode = $head_code . "\n"; + } + + # Variables. + my %offsets; + my @instrs; + my %num_references; + my $group_size = 999; + + # Do basic error checking. Associate operands of instructions + # with the correct micro instructions. Calculate offsets for micro + # instructions. + foreach my $ref_instr (@in_instrs) { + my($specific,$arity,@subs) = @$ref_instr; + my $specific_key = "$specific/$arity"; + my $specific_op_ref = $specific_op{$specific_key}; + error("no $specific_key instruction") + unless defined $specific_op_ref; + foreach my $specific_op (@$specific_op_ref) { + my($name, $hotness, @args) = @{$specific_op}; + $group_hotness = $hotness unless $group_hotness >= $hotness; + my $offset = 0; + my @rest = @args; + my @new_subs; + my $print_name = print_name($specific, @args); + my $opcase = $print_name; + my $last = $subs[$#subs]; + foreach my $s (@subs) { + my $code = $c_code{$s}; + my(undef,undef,@c_args) = @{$code}; + my @first; + foreach (0..$#c_args) { + push @first, shift @rest; + } + my $size = cg_combined_size($s, 1, @first); + $offsets{$s} = $offset + unless defined $offsets{$s} and $offsets{$s} < $offset; + $offset += $size - 1; + my $label = micro_label($s); + $num_references{$label} = 0; + push @new_subs, [$opcase,$label,$s,$size-1,@first]; + $opcase = ''; + } + $spec_op_info{$print_name}->{'size'} = $offset + 1; + $group_size = $offset if $group_size >= $offset; + push @instrs, [$specific_key,@new_subs]; + } + } - # The following argument types should be included as macro arguments. - my(%incl_arg) = ('c' => 1, - 'i' => 1, - 'a' => 1, - 'A' => 1, - 'N' => 1, - 'U' => 1, - 'I' => 1, - 't' => 1, - 'P' => 1, - 'Q' => 1, - ); + # Link the sub instructions for each instructions to each + # other. + my @all_instrs; + foreach my $instr (@instrs) { + my($specific_key,@subs) = @{$instr}; + for (my $i = 0; $i < @subs; $i++) { + my($opcase,$label,$s,$size,@args) = @{$subs[$i]}; + my $next = ''; + (undef,$next) = @{$subs[$i+1]} if $i < $#subs; + $num_references{$next}++ if $next; + my $instr_info = "$opcase:$label:$next:$s:$size:@args"; + push @all_instrs, [$label,$offsets{$s},$instr_info]; + } + } - # Pick up the macro to use and its flags (if any). + my %order_to_instrs; + my %label_to_offset; + my %order_to_offset; + foreach my $instr (@all_instrs) { + my($label,$offset,$instr_info) = @$instr; + my $sort_key = sprintf("%02d.%02d", $offset, $num_references{$label}); + push @{$order_to_instrs{$sort_key}}, $instr_info; + $label_to_offset{$label} = $offset; + $order_to_offset{$sort_key} = $offset; + } + + my(@slots) = sort {$a <=> $b} keys %order_to_instrs; + + # Now generate the code for the entire group. + my $offset = 0; + my @opcase_labels; + my %down; + my %up; + for(my $i = 0; $i < @slots; $i++) { + my $key = $slots[$i]; + + # Sort micro-instructions with OpCase before other micro-instructions. + my(@instrs) = @{$order_to_instrs{$key}}; + my $order_func = sub { + my $a_key = ($a =~ /^:/) ? "1$a" : "0$a"; + my $b_key = ($b =~ /^:/) ? "1$b" : "0$b"; + $a_key cmp $b_key; + }; + @instrs = sort $order_func @instrs; + + my %seen; + foreach my $instr (@instrs) { + my($opcase,$label,$next,$s,$size,$args) = split ":", $instr; + my(@first) = split " ", $args; + + my $seen_key = "$label:$next:" . scalar(@first); + next if $opcase eq '' and $seen{$seen_key}; + $seen{$seen_key} = 1; + $seen_key .= $opcase; + + if ($opcase ne '') { + $gcode .= "OpCase($opcase):\n"; + push @opcase_labels, $opcase; + } + if ($num_references{$label}) { + $gcode .= "$label:\n"; + } + + my $flags = ''; + my $transfer_to_next = ''; + my $inc = 0; + + unless ($i == $#slots) { + $flags = "-no_next"; + my $next_offset = $label_to_offset{$next}; + $inc = ($offset + $size) - $next_offset; + $transfer_to_next = "I += $inc;\n" if $inc; + $transfer_to_next .= "goto $next;\n\n"; + } + + my($gen_code,$down,$up) = + cg_combined_code($s, 1, $flags, $offset, + $group_size-$offset, $inc, @first); + my $spec_label = "$opcase$label"; + $down{$spec_label} = $down; + $up{$spec_label} = $up; + $gcode .= $gen_code . $transfer_to_next; + } + $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots; + } - $macro = $macro{$name} if defined $macro{$name}; - $flags = $macro_flags{$name} if defined $macro_flags{$name}; + foreach my $print_name (@opcase_labels) { + my $info = $spec_op_info{$print_name}; + $info->{'adj'} = $info->{'size'} - $group_size - 1; + } # - # Add any arguments to be included as macro arguments (for instance, - # 'p' is usually not an argument, except for calls). + # Assemble pack specifications for all instructions in the group. # + foreach my $instr (@instrs) { + my(undef,@subs) = @{$instr}; + my $down = ''; + my $up = ''; + for (my $i = 0; $i < @subs; $i++) { + my($opcase,$label) = @{$subs[$i]}; + my $spec_label = "$opcase$label"; + if (defined $down{$spec_label}) { + $down = $down{$spec_label} . $down; + $up = $up . $up{$spec_label}; + } + } + my $print_name = $subs[0]->[0]; + my $info = $spec_op_info{$print_name}; + $info->{'pack_spec'} = build_pack_spec("$down:$up"); + } - while ($flags =~ /-arg_(\w)/g) { - $incl_arg{$1} = 1; - }; + ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels); +} + +sub micro_label { + my $label = shift; + $label =~ s/[.]/__/g; + $label; +} + + +# +# Basic code generation for one instruction. +# + +sub cg_basic { + my($name,@args) = @_; + my($size,$code,$pack_spec) = code_gen($name, 1, '', 0, undef, undef, @args); + $pack_spec = build_pack_spec($pack_spec); + ($size,$code,$pack_spec); +} + +# +# Calculate size for a micro instruction. +# + +sub cg_combined_size { + my($name,$pack,@args) = @_; + my($size) = code_gen($name, $pack, '', 0, undef, undef, @args); + $size; +} + +# +# Generate code for a micro instruction. +# + +sub cg_combined_code { + my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_; + my($size,$code,$pack_spec) = code_gen(@_); + if ($pack_spec eq '') { + ($code,'',''); + } else { + my($down,$up) = split /:/, $pack_spec; + ($code,$down,$up); + } +} + +sub code_gen { + my($name,$pack,$extra_comments,$offset,$comp_size,$inc,@args) = @_; + my $group_size = defined $comp_size ? $comp_size + $inc : undef; + my $size = 0; + my $flags = ''; + my @f; + my $prefix = ''; + my $tmp_arg_num = 1; + my $pack_spec = ''; + my $var_decls = ''; # - # Pack arguments if requested. + # Pack arguments for hot code with an implementation. # - if ($flags =~ /-pack/ && $hot) { - ($prefix, $pack_spec, @args) = do_pack(@args); + my $c_code_ref = $c_code{$name}; + if ($pack and defined $c_code_ref and $name ne 'catch') { + ($var_decls, $pack_spec, @args) = do_pack($offset, @args); } # @@ -993,259 +1335,503 @@ sub basic_generator { # the macro. # + my $need_block = 0; + my $arg_offset = $offset; foreach (@args) { my($this_size) = $arg_size{$_}; SWITCH: { - /^pack:(\d):(.*)/ and do { push(@f, $2); - push(@f_types, 'packed'); - $this_size = $1; - last SWITCH; - }; - /r/ and do { push(@f, "r(0)"); push(@f_types, $_); last SWITCH }; - /[xy]/ and do { push(@f, "$_" . "b(Arg($size))"); - push(@f_types, $_); - last SWITCH; - }; - /n/ and do { push(@f, "NIL"); push(@f_types, $_); last SWITCH }; - /s/ and do { my($tmp) = "targ$tmp_arg_num"; - $var_decls .= "Eterm $tmp; "; - $tmp_arg_num++; - push(@f, $tmp); - push(@f_types, $_); - $prefix .= "GetR($size, $tmp);\n"; - last SWITCH; }; - /d/ and do { $var_decls .= "Eterm dst; Eterm* dst_ptr; "; - push(@f, "*dst_ptr"); - push(@f_types, $_); - $prefix .= "dst = Arg($size);\n"; - $prefix .= "dst_ptr = REG_TARGET_PTR(dst);\n"; - last SWITCH; - }; - defined($incl_arg{$_}) - and do { push(@f, "Arg($size)"); - push(@f_types, $_); - last SWITCH; - }; - - /[fp]/ and do { $fail_type = $_; last SWITCH }; - - /[eLIFEbASjPowlq]/ and do { last SWITCH; }; + /^packed:d:(\d):(.*)/ and do { + $var_decls .= "Eterm dst = $2;\n" . + "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; + push(@f, "*dst_ptr"); + $this_size = $1; + last SWITCH; + }; + /^packed:[a-zA-z]:(\d):(.*)/ and do { + push(@f, $2); + $this_size = $1; + last SWITCH; + }; + /r/ and do { + push(@f, "r(0)"); + last SWITCH; + }; + /[lxyS]/ and do { + push(@f, $_ . "b(" . arg_offset($arg_offset) . ")"); + last SWITCH; + }; + /n/ and do { + push(@f, "NIL"); + last SWITCH; + }; + /s/ and do { + my($tmp) = "targ$tmp_arg_num"; + $var_decls .= "Eterm $tmp;\n"; + $tmp_arg_num++; + push(@f, $tmp); + $prefix .= "GetR($arg_offset, $tmp);\n"; + $need_block = 1; + last SWITCH; + }; + /d/ and do { + $var_decls .= "Eterm dst = " . arg_offset($arg_offset) . ";\n" . + "Eterm* dst_ptr = REG_TARGET_PTR(dst);\n"; + push(@f, "*dst_ptr"); + last SWITCH; + }; + defined $arg_size{$_} and do { + push @f, arg_offset($arg_offset); + last SWITCH; + }; die "$name: The generator can't handle $_, at"; } $size += $this_size; + $arg_offset += $this_size; } # - # Add a fail action macro if requested. + # If the implementation is in beam_emu.c, there is nothing + # more to do. # + unless (defined $c_code_ref) { + return ($size+1, undef, ''); + } - $flags =~ /-fail_action/ and do { - $no_prefetch = 1; - if (!defined $fail_type) { - my($i); - for ($i = 0; $i < @f_types; $i++) { - local($_) = $f_types[$i]; - /[rxycians]/ and do { push(@f, "Badmatch($f[$i])"); next }; - } - } elsif ($fail_type eq 'f') { - push(@f, "ClauseFail()"); - } else { - my($i); - for ($i = 0; $i < @f_types; $i++) { - local($_) = $f_types[$i]; - /[rxycians]/ and do { push(@f, "Badmatch($f[$i])"); next }; - } - } - }; + $group_size = $size unless defined $group_size; # - # Add a size argument if requested. + # Generate main body of the implementation. # + my($c_code,$where,@c_args) = @{$c_code_ref}; + my %bindings; + $c_code_used{$name} = 1; - $flags =~ /-size/ and do { - push(@f, $size); - }; + if (@f != @c_args) { + error("$where: defining '$name' with ", scalar(@c_args), + " arguments instead of expected ", scalar(@f), " arguments"); + } - # Generate the macro if requested. - my($code); - if (defined $macro{$name}) { - my($macro_code) = "$prefix$macro(" . join(', ', @f) . ");"; - $var_decls .= "BeamInstr tmp_packed1;" - if $macro_code =~ /tmp_packed1/; - $var_decls .= "BeamInstr tmp_packed2;" - if $macro_code =~ /tmp_packed2/; - if ($flags =~ /-nonext/) { - $code = join("\n", - "{ $var_decls", - $macro_code, - "}"); - } elsif ($flags =~ /-goto:(\S*)/) { - my $goto = $1; - $code = join("\n", - "{ $var_decls", - $macro_code, - "I += $size + 1;", - "goto $goto;", - "}"); - } elsif ($no_prefetch) { - $code = join("\n", - "{ $var_decls", - $macro_code, - "Next($size);", - "}", ""); - } else { - $code = join("\n", - "{ $var_decls", - "BeamInstr* next;", - "PreFetch($size, next);", - "$macro_code", - "NextPF($size, next);", - "}", ""); - } + for (my $i = 0; $i < @f; $i++) { + my $var = $c_args[$i]; + $bindings{$var} = $f[$i]; + } + $bindings{'NEXT_INSTRUCTION'} = "I+" . ($group_size+$offset+1); + $bindings{'IP_ADJUSTMENT'} = defined $inc ? $inc : 0; + $c_code = eval { expand_all($c_code, \%bindings) }; + unless (defined $c_code) { + warn $@; + error("... from the body of $name at $where"); + } + my(@comments) = $c_code =~ m@//[|]\s*(.*)@g; + $c_code =~ s@//[|]\s*(.*)\n?@@g; + $flags = "@comments $extra_comments"; + + # + # Generate code for transferring to the next instruction. + # + my $dispatch_next; + my $instr_offset = $group_size + $offset + 1; + + if ($flags =~ /-no_next/) { + $dispatch_next = ""; + } elsif ($flags =~ /-no_prefetch/) { + $dispatch_next = "\nI += $instr_offset;\n" . + "ASSERT(VALID_INSTR(*I));\n" . + "Goto(*I);"; + } else { + $var_decls .= "BeamInstr next_pf = I[$instr_offset];\n"; + $dispatch_next = "\nI += $instr_offset;\n" . + "ASSERT(VALID_INSTR(next_pf));\n" . + "Goto(next_pf);"; + } + + # + # Assemble the complete code for the instruction. + # + my $body = "$c_code$dispatch_next"; + if ($need_block) { + $body = "$prefix\{\n$body\n}"; + } else { + $body = "$prefix$body"; + } + my $code = join("\n", + "{", + "$var_decls$body", + "}", ""); + ($size+1, $code, $pack_spec); +} + +sub arg_offset { + my $offset = shift; + "I[" . ($offset+1) . "]"; +} + +sub expand_all { + my($code,$bindings_ref) = @_; + my %bindings = %{$bindings_ref}; + + # Expand all $Var occurrences. + $code =~ s/[\$](\w[\w\d]*)(?!\()/defined $bindings{$1} ? $bindings{$1} : "\$$1"/ge; + + # Find calls to macros, $name(...), and expand them. + my $res = ""; + while ($code =~ /[\$](\w[\w\d]*)\(/) { + my $macro_name = $1; + my $keep = substr($code, 0, $-[0]); + my $after = substr($code, $+[0]); + + my $body; + ($body,$code) = expand_macro($macro_name, $after, \%bindings); + $res .= "$keep$body"; + } + + $res . $code; +} + +sub expand_macro { + my($name,$rest,$bindings_ref) = @_; + + my $c_code = $c_code{$name}; + defined $c_code or + error("calling undefined macro '$name'..."); + $c_code_used{$name} = 1; + my ($body,$where,@vars) = @{$c_code}; + + # Separate the arguments into @args; + my @args; + my $level = 1; + my %inc = ('(' => 1, ')' => -1, + '[' => 1, ']' => -1, + '{' => 1, '}' => -1); + my $arg = undef; + while ($rest =~ /([,\(\[\{\}\]\)]|([^,\(\[\{\}\]\)]*))/g) { + my $token = $1; + my $inc = $inc{$token} || 0; + $level += $inc; + if ($level == 0) { + $rest = substr($rest, pos($rest)); + push @args, $arg if defined $arg; + last; + } + if ($token eq ',') { + if ($level == 1) { + push @args, $arg; + $arg = ""; + } + next; + } + $arg .= $token; + } + + # Trim leading whitespace from each argument. + foreach my $arg (@args) { + $arg =~ s/^\s*//; + } + + # Make sure that the number of arguments are correct. + if (@vars != @args) { + error("calling $name with ", scalar(@args), + " arguments instead of expected ", scalar(@vars), " arguments..."); + } + + # Now combine bindings from the parameter names and arguments. + my %bindings = %{$bindings_ref}; + my %new_bindings; + + # Keep the special, pre-defined bindings. + foreach my $key (qw(NEXT_INSTRUCTION IP_ADJUSTMENT)) { + $new_bindings{$key} = $bindings{$key}; + } + + for (my $i = 0; $i < @vars; $i++) { + my $arg = $args[$i]; + $arg = eval { expand_all($arg, \%bindings) }; + unless (defined $arg) { + warn $@; + die "... from the body of $name at $where\n"; + } + $new_bindings{$vars[$i]} = $arg; + } + + $body = eval { expand_all($body, \%new_bindings) }; + unless (defined $body) { + warn $@; + die "... from the body of $name at $where\n"; + } + + # Handle built-in macros. + if ($name eq 'ARG_POSITION') { + if ($body =~ /^I\[(\d+)\]$/) { + $body = $1; + } else { + $body = 0; + } + } elsif ($name eq 'IS_PACKED') { + $body = ($body =~ /^I\[\d+\]$/) ? 0 : 1; } - # Return the size and code for the macro (if any). - $size++; - ($size, $code, $pack_spec); + # Wrap body if needed and return resul.t + $body = "do {\n$body\n} while (0)" + if needs_do_wrapper($body); + ($body,$rest); +} + +# Conservative heuristic to determine whether a do { ... } while(0) +# wrapper is needed. +sub needs_do_wrapper { + local $_ = shift; + + s@^//[|][^\n]*\n@@; + s@^\s*@@s; + s@^/[*].*[*]/\s*@@s; + return 1 if /^(Eterm|Uint|Sint|int|unsigned)/; # Definitely needed. + return 0 if /^do/; + return 0 if /^SET_I/; + return 0 if /^SET_CP/; + return 0 if /^ERTS_NO_FPE_CHECK_INIT/; + return 0 if /^ASSERT/; + return 0 if /^DTRACE/; + return 0 if /^[A-Za-z_]*\s*=/; + return 0 if /^c_p->/; + return 0 if /^[A-Z_]*SWAPOUT/; + return 0 if /^if\s*[(]/; + return 0 if /^goto\b/; + return 0 if /^\d+/; + return 1; # Not sure, say that it is needed. } sub do_pack { - my(@args) = @_; + my($offset,@args) = @_; my($packable_args) = 0; - my @is_packable; # Packability (boolean) for each argument. - my $wide_packing = 0; - my(@orig_args) = @args; + my @bits_needed; # Bits needed for each argument. # - # Count the number of packable arguments. If we encounter any 's' or 'd' - # arguments, packing is not possible. + # Define the minimum number of bits needed for the packable argument types. + # + my %bits_needed = ('x' => 10, + 'y' => 10, + 'Q' => 10, + 'l' => 10, + 'S' => 16, + 'd' => 16, + 't' => 16); + if ($wordsize == 64) { + $bits_needed{'I'} = 32; + } + + # + # Count the number of packable arguments. # - my $packable_types = "xytQ"; foreach my $arg (@args) { - if ($arg =~ /^[$packable_types]/) { + if (defined $bits_needed{$arg}) { $packable_args++; - push @is_packable, 1; - } elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) { - $wide_packing = 1; - push @is_packable, 1; - if (++$packable_args == 2) { - # We can only pack two arguments. Turn off packing - # for the rest of the arguments. - $packable_types = "\xFF"; - } - } elsif ($arg =~ /^[sd]/) { - return ('', '', @args); - } elsif ($arg =~ /^[scq]/ and $packable_args > 0) { - # When packing, this operand will be picked up from the - # code array, put onto the packing stack, and later put - # back into a different location in the code. The problem - # is that if this operand is a literal, the original - # location in the code would have been remembered in a - # literal patch. For packing to work, we would have to - # adjust the position in the literal patch. For the - # moment, adding additional instructions to the packing - # engine to handle this does not seem worth it, so we will - # just turn off packing. - return ('', '', @args); + push @bits_needed, $bits_needed{$arg}; } else { - push @is_packable, 0; + push @bits_needed, 0; } } # - # Get out of here if too few or too many arguments. + # Try to pack 'f' and 'j', but not at expense at worse packing + # for other operands. For example, given the arguments "f x x", we + # want the 'x' operands to be packed, not 'f' and 'x' packed and + # the final 'x' not packed. # - return ('', '', @args) if $packable_args < 2; - my($size) = 0; - my($pack_prefix) = ''; - my($down) = ''; # Pack commands (towards instruction - # beginning). - my($up) = ''; # Pack commands (storing back while - # moving forward). + if ($wordsize == 64 and $packable_args == 1) { + for (my $i = 0; $i < @args; $i++) { + if ($args[$i] =~ /^[fj]$/) { + $bits_needed[$i] = 32; + $packable_args++; + last; + } + } + } - my $args_per_word = $args_per_word[$packable_args]; - my @shift; - my @mask; - my @instr; + # + # Nothing to pack unless there are at least 2 packable arguments. + # + return ('', ':', @args) if $packable_args < 2; - if ($wide_packing) { - @shift = ('0', 'BEAM_WIDE_SHIFT'); - @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); - @instr = ('w', 'i'); - } else { - @shift = @{$pack_shift[$args_per_word]}; - @mask = @{$pack_mask[$args_per_word]}; - @instr = @{$pack_instr[$args_per_word]}; + # + # Determine how many arguments we should pack into each word. + # + my @args_per_word; + my @need_wide_mask; + my $bits = 0; + my $word = 0; + $args_per_word[0] = 0; + $need_wide_mask[0] = 0; + for (my $i = 0; $i < @args; $i++) { + if ($bits_needed[$i]) { + my $needed = $bits_needed[$i]; + + my $next_word = sub { + $word++; + $args_per_word[$word] = 0; + $need_wide_mask[$word] = 0; + $bits = 0; + }; + + if ($bits+$needed > $wordsize) { # Does not fit. + $next_word->(); + } + if ($args_per_word[$word] == 4) { # Can't handle more than 4 args. + $next_word->(); + } + if ($needed == 32 and $args_per_word[$word] > 1) { + # Must only pack two arguments in this word, and there + # are already at least two arguments here. + $next_word->(); + } + $args_per_word[$word]++; + $bits += $needed; + if ($needed == 32) { + $need_wide_mask[$word]++; + } + if ($need_wide_mask[$word] and $bits > 32) { + # Can only pack two things in a word where one + # item is 32 bits. Force the next item into + # the next word. + $bits = $wordsize; + } + } } # + # Try to balance packing between words. + # + if ($args_per_word[$#args_per_word] == 1) { + if ($args_per_word[$#args_per_word-1] < 3) { + pop @args_per_word; + } else { + $args_per_word[$#args_per_word-1]--; + $args_per_word[$#args_per_word]++; + } + } elsif (@args_per_word == 2 and + $args_per_word[0] == 4 and + $args_per_word[1] == 2) { + $args_per_word[0] = 3; + $args_per_word[1] = 3; + } elsif (@args_per_word == 2 and + $args_per_word[0] == 3 and + $args_per_word[1] == 1) { + $args_per_word[0] = 2; + $args_per_word[1] = 2; + } + + my $size = 0; + my $pack_prefix = ''; + my $down = ''; # Pack commands (towards instruction + # beginning). + my $up = ''; # Pack commands (storing back while + # moving forward). + + # Skip an unpackable argument. + my $skip_unpackable = sub { + my($arg) = @_; + + if ($arg_size{$arg}) { + # Save the argument on the pack engine's stack. + my $push = 'g'; + if ($type_bit{$arg} & $type_bit{'q'}) { + # The operand may be a literal. + $push = 'q'; + } elsif ($type_bit{$arg} & $type_bit{'f'}) { + # The operand may be a failure label. + $push = 'f'; + } + $down = "$push${down}"; + $up = "${up}p"; + } + }; + + # # Now generate the packing instructions. One complication is that # the packing engine works from right-to-left, but we must generate # the instructions from left-to-right because we must calculate # instruction sizes from left-to-right. - # - # XXX Packing 3 't's in one word won't work. Sorry. - my $did_some_packing = 0; # Nothing packed yet. - my($ap) = 0; # Argument number within word. - my($tmpnum) = 1; # Number of temporary variable. - my($expr) = ''; - for (my $i = 0; $i < @args; $i++) { - my($reg) = $args[$i]; - my($this_size) = $arg_size{$reg}; - if ($is_packable[$i]) { - $this_size = 0; - $did_some_packing = 1; - - if ($ap == 0) { - $pack_prefix .= "tmp_packed$tmpnum = Arg($size);\n"; - $up .= "p"; - $down = "P$down"; - $this_size = 1; - } - - $down = "$instr[$ap]$down"; - my($unpack) = make_unpack($tmpnum, $shift[$ap], $mask[$ap]); - $args[$i] = "pack:$this_size:$reg" . "b($unpack)"; + my $arg_num = 0; + for (my $word = 0; $word < @args_per_word; $word++) { + my $ap = 0; # Argument number within word. + my $packed_var = "tmp_packed" . ($word+1); + my $args_per_word = $args_per_word[$word]; + my @shift; + my @mask; + my @instr; + + if ($need_wide_mask[$word]) { + @shift = ('0', 'BEAM_WIDE_SHIFT'); + @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); + @instr = ('w', 'w'); + } else { + @shift = @{$pack_shift[$args_per_word]}; + @mask = @{$pack_mask[$args_per_word]}; + @instr = @{$pack_instr[$args_per_word]}; + } - if (++$ap == $args_per_word) { - $ap = 0; - $tmpnum++; - } - } elsif ($arg_size{$reg} && $did_some_packing) { - # - # This is an argument that can't be packed. Normally, we must - # save it on the pack engine's stack, unless: - # - # 1. The argument has zero size (e.g. r(0)). Such arguments - # will not be loaded. They disappear. - # 2. If the argument is on the left of the first packed argument, - # the packing engine will never access it (because the engine - # operates from right-to-left). - # + while ($ap < $args_per_word) { + my $reg = $args[$arg_num]; + my $this_size = $arg_size{$reg}; + if ($bits_needed[$arg_num]) { + $this_size = 0; + + if ($ap == 0) { + $pack_prefix .= "Eterm $packed_var = " . + arg_offset($size+$offset) . ";\n"; + $up .= "p"; + $down = "P$down"; + $this_size = 1; + } + + $down = "$instr[$ap]$down"; + my $unpack = make_unpack($packed_var, $shift[$ap], $mask[$ap]); + $args[$arg_num] = "packed:$reg:$this_size:$reg" . "b($unpack)"; + + $ap++; + } else { + $skip_unpackable->($reg); + } + $size += $this_size; + $arg_num++; + } + } - $down = "g${down}"; - $up = "${up}p"; - } - $size += $this_size; + # + # Skip any unpackable arguments at the end. + # + while ($arg_num < @args) { + $skip_unpackable->($args[$arg_num]); + $arg_num++; } - my $pack_spec = $down . $up; + my $pack_spec = "$down:$up"; return ($pack_prefix, $pack_spec, @args); } sub make_unpack { - my($tmpnum, $shift, $mask) = @_; + my($packed_var, $shift, $mask) = @_; - my($e) = "tmp_packed$tmpnum"; + my $e = $packed_var; $e = "($e>>$shift)" if $shift; $e .= "&$mask" unless $mask eq $WHOLE_WORD; $e; } +sub build_pack_spec { + my $pack_spec = shift; + return '' if $pack_spec eq ''; + my($down,$up) = split /:/, $pack_spec; + while ($down =~ /[gfq]$/ and $up =~ /^p/) { + $down = substr($down, 0, -1); + $up = substr($up, 1); + } + "$down$up"; +} + sub quote { local($_) = @_; return "'$_'" if $_ eq 'try'; @@ -1286,8 +1872,11 @@ sub parse_transformation { # my @to; - if ($to =~ /^(\w+)\((.*?)\)/) { - my($name, $arglist) = ($1, $2); + if ($to =~ /^(\w+)\((.*?)\)(.*)/) { + my($name, $arglist, $garbage) = ($1, $2, $3); + if ($garbage =~ /\S/) { + error("garbage after call to '$name()'"); + } @to = (compile_transform_function($name, split(/\s*,\s*/, $arglist))); } else { @to = split(/\s*\|\s*/, $to); diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index 47e1528958..094a35ae4b 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -59,7 +59,6 @@ my %dirty_bif_tab; my @bif; my @bif_info; -my $dirty_schedulers = 'no'; my $dirty_schedulers_test = 'no'; my $hipe = 'no'; @@ -73,10 +72,6 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { $include = shift; die "No directory for -include argument specified" unless defined $include; - } elsif($opt eq '-ds') { - $dirty_schedulers = shift; - die "No -ds argument specified" - unless defined $dirty_schedulers; } elsif($opt eq '-dst') { $dirty_schedulers_test = shift; die "No -dst argument specified" @@ -140,21 +135,19 @@ while (<>) { push(@bif_info, [$type, $sched_type, $alias3, $alias]); } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io' or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') { - if ($dirty_schedulers eq 'yes') { - my($bif,$other) = (@args); - $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); - my($mod,$name,$arity) = ($1,$2,$3); - my $mfa = "$mod:$name/$arity"; - if (($type eq 'dirty-cpu') - or (($dirty_schedulers_test eq 'yes') - and ($type eq 'dirty-cpu-test'))) { - $dirty_bif_tab{$mfa} = 'dirty_cpu'; - } elsif (($type eq 'dirty-io') - or (($dirty_schedulers_test eq 'yes') - and ($type eq 'dirty-io-test'))) { - $dirty_bif_tab{$mfa} = 'dirty_io'; - } - } + my($bif,$other) = (@args); + $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); + my($mod,$name,$arity) = ($1,$2,$3); + my $mfa = "$mod:$name/$arity"; + if (($type eq 'dirty-cpu') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-cpu-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_cpu'; + } elsif (($type eq 'dirty-io') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-io-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_io'; + } } else { error("invalid line"); } diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0 index fcde4a0123..29f2d3d62d 100644 --- a/erts/emulator/valgrind/suppress.patched.3.6.0 +++ b/erts/emulator/valgrind/suppress.patched.3.6.0 @@ -374,3 +374,10 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard index bb07c92fc1..99a3ee4048 100644 --- a/erts/emulator/valgrind/suppress.standard +++ b/erts/emulator/valgrind/suppress.standard @@ -342,3 +342,11 @@ fun:erts_debug_set_internal_state_2 fun:process_main } +{ +Thread specific dlerror buffer. Either bug in libc or valgrind. +Memcheck:Leak +... +fun:_dlerror_run +... +} + diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index 583426460e..1f35cef669 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -56,8 +56,8 @@ ERTS_INCL = -I$(ERL_TOP)/erts/include \ CC = @CC@ WFLAGS = @WFLAGS@ -CFLAGS = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) @WFLAGS@ -I$(SYSDIR) -I$(EMUDIR) \ - $(ERTS_INCL) -DOTP_SYSTEM_VERSION=\"$(SYSTEM_VSN)\" +CFLAGS = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) @WFLAGS@ -I$(SYSOSDIR) -I$(EMUDIR) -I. \ + -I$(COMSYSDIR) $(ERTS_INCL) -DOTP_SYSTEM_VERSION=\"$(SYSTEM_VSN)\" LD = @LD@ LIBS = @LIBS@ LDFLAGS = @LDFLAGS@ @@ -69,16 +69,20 @@ endif ifeq ($(TARGET),win32) ifeq ($(TYPE),debug) -CFLAGS = $(subst -O2,-g,@CFLAGS@ @DEFS@ $(TYPE_FLAGS) @WFLAGS@ -I$(SYSDIR) \ - -I$(EMUDIR) $(ERTS_INCL) -DOTP_SYSTEM_VERSION=\"$(SYSTEM_VSN)\") +CFLAGS = $(subst -O2,-g,@CFLAGS@ @DEFS@ $(TYPE_FLAGS) @WFLAGS@ -I$(SYSOSDIR) \ + -I$(EMUDIR) -I$(COMSYSDIR) $(ERTS_INCL) \ + -DOTP_SYSTEM_VERSION=\"$(SYSTEM_VSN)\") LDFLAGS += -g endif endif + BINDIR = $(ERL_TOP)/bin/$(TARGET) OBJDIR = $(ERL_TOP)/erts/obj$(TYPEMARKER)/$(TARGET) EMUDIR = $(ERL_TOP)/erts/emulator/beam +COMSYSDIR = $(ERL_TOP)/erts/emulator/sys/common EMUOSDIR = $(ERL_TOP)/erts/emulator/@ERLANG_OSTYPE@ -SYSDIR = $(ERL_TOP)/erts/emulator/sys/@ERLANG_OSTYPE@ +SYSDIR = $(ERL_TOP)/erts/emulator/sys/common +SYSOSDIR = $(ERL_TOP)/erts/emulator/sys/@ERLANG_OSTYPE@ DRVDIR = $(ERL_TOP)/erts/emulator/drivers/@ERLANG_OSTYPE@ UXETC = ../unix WINETC = ../win32 diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c index 6639c83778..efa7ac3493 100644 --- a/erts/etc/common/ct_run.c +++ b/erts/etc/common/ct_run.c @@ -20,16 +20,7 @@ /* * Purpose: Common Test front-end. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#ifdef __WIN32__ -#include <winbase.h> -#endif - -#include <ctype.h> +#include "etc_common.h" #define NO 0 #define YES 1 diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c index c8d977f6de..0e74eb065b 100644 --- a/erts/etc/common/dialyzer.c +++ b/erts/etc/common/dialyzer.c @@ -20,16 +20,8 @@ /* * Purpose: Dialyzer front-end. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#ifdef __WIN32__ -#include <winbase.h> -#endif -#include <ctype.h> +#include "etc_common.h" #define NO 0 #define YES 1 diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c index cbbd2a37cd..8cfd98bcc4 100644 --- a/erts/etc/common/erlc.c +++ b/erts/etc/common/erlc.c @@ -20,19 +20,7 @@ /* * Purpose: Common compiler front-end. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#ifdef __WIN32__ -#include <winbase.h> -/* FIXE ME config_win32.h? */ -#define HAVE_STRERROR 1 -#define snprintf _snprintf -#endif - -#include <ctype.h> +#include "etc_common.h" #define NO 0 #define YES 1 diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 51ed2d0dff..d61a3cbf95 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -23,14 +23,9 @@ * additions required for Windows NT. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif +#include "etc_common.h" -#include "sys.h" #include "erl_driver.h" -#include <stdlib.h> -#include <stdarg.h> #include "erl_misc_utils.h" #ifdef __WIN32__ @@ -194,9 +189,7 @@ void error(char* format, ...); * Local functions. */ -#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_PLAIN_EMU) static void usage_notsup(const char *switchname, const char *alt); -#endif static char **build_args_from_env(char *env_var); static char **build_args_from_string(char *env_var); static void initial_argv_massage(int *argc, char ***argv); @@ -244,7 +237,7 @@ static int verbose = 0; /* If non-zero, print some extra information. */ static int start_detached = 0; /* If non-zero, the emulator should be * started detached (in the background). */ -static int start_smp_emu = 0; /* Start the smp emulator. */ +static int start_smp_emu = 1; /* Start the smp emulator. */ static const char* emu_type = 0; /* Type of emulator (lcnt, valgrind, etc) */ #ifdef __WIN32__ @@ -467,10 +460,6 @@ int main(int argc, char **argv) * Construct the path of the executable. */ cpuinfo = erts_cpu_info_create(); - /* '-smp auto' is default */ -#ifdef ERTS_HAVE_SMP_EMU - start_smp_emu = 1; -#endif #if defined(__WIN32__) && defined(WIN32_ALWAYS_DEBUG) emu_type = "debug"; @@ -502,23 +491,13 @@ int main(int argc, char **argv) i++; smp_enable: ; -#if !defined(ERTS_HAVE_SMP_EMU) - usage_notsup("-smp enable", ""); -#endif } else if (strcmp(argv[i+1], "disable") == 0) { i++; smp_disable: -#ifdef ERTS_HAVE_PLAIN_EMU - start_smp_emu = 0; -#else usage_notsup("-smp disable", " Use \"+S 1\" instead."); -#endif } else { smp: ; -#if !defined(ERTS_HAVE_SMP_EMU) - usage_notsup("-smp", ""); -#endif } } else if (strcmp(argv[i], "-smpenable") == 0) { goto smp_enable; @@ -831,6 +810,28 @@ int main(int argc, char **argv) add_Eargs(argv[i+1]); i++; break; + case 'I': + if (argv[i][2] == 'O' && (argv[i][3] == 't' || argv[i][3] == 'p')) { + if (argv[i][4] != '\0') + goto the_default; + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + break; + } + if (argv[i][2] == 'O' && argv[i][3] == 'P' && + (argv[i][4] == 't' || argv[i][4] == 'p')) { + if (argv[i][5] != '\0') + goto the_default; + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + break; + } + usage(argv[i]); + break; case 'S': if (argv[i][2] == 'P') { if (argv[i][3] != '\0') @@ -886,8 +887,8 @@ int main(int argc, char **argv) case 'c': argv[i][0] = '-'; if (argv[i][2] == '\0' && i+1 < argc) { - if (sys_strcmp(argv[i+1], "true") == 0 - || sys_strcmp(argv[i+1], "false") == 0) { + if (strcmp(argv[i+1], "true") == 0 + || strcmp(argv[i+1], "false") == 0) { add_Eargs(argv[i]); add_Eargs(argv[i+1]); i++; @@ -1159,15 +1160,6 @@ usage_aux(void) #ifdef __WIN32__ "[-start_erl [datafile]] " #endif - "[-smp [auto" -#ifdef ERTS_HAVE_SMP_EMU - "|enable" -#endif -#ifdef ERTS_HAVE_PLAIN_EMU - "|disable" -#endif - "]" - "] " "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] [-start_epmd BOOLEAN] " "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c [BOOLEAN]] " "[+C MODE] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] " @@ -1188,14 +1180,12 @@ usage(const char *switchname) usage_aux(); } -#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_PLAIN_EMU) static void usage_notsup(const char *switchname, const char *alt) { fprintf(stderr, "Argument \'%s\' not supported.%s\n", switchname, alt); usage_aux(); } -#endif static void usage_format(char *format, ...) @@ -2195,18 +2185,18 @@ static WCHAR *utf8_to_utf16(unsigned char *bytes) res = target = emalloc((num + 1) * sizeof(WCHAR)); while (*bytes) { if (((*bytes) & ((unsigned char) 0x80)) == 0) { - unipoint = (Uint) *bytes; + unipoint = (unsigned int) *bytes; ++bytes; } else if (((*bytes) & ((unsigned char) 0xE0)) == 0xC0) { unipoint = - (((Uint) ((*bytes) & ((unsigned char) 0x1F))) << 6) | - ((Uint) (bytes[1] & ((unsigned char) 0x3F))); + (((unsigned int) ((*bytes) & ((unsigned char) 0x1F))) << 6) | + ((unsigned int) (bytes[1] & ((unsigned char) 0x3F))); bytes += 2; } else if (((*bytes) & ((unsigned char) 0xF0)) == 0xE0) { unipoint = - (((Uint) ((*bytes) & ((unsigned char) 0xF))) << 12) | - (((Uint) (bytes[1] & ((unsigned char) 0x3F))) << 6) | - ((Uint) (bytes[2] & ((unsigned char) 0x3F))); + (((unsigned int) ((*bytes) & ((unsigned char) 0xF))) << 12) | + (((unsigned int) (bytes[1] & ((unsigned char) 0x3F))) << 6) | + ((unsigned int) (bytes[2] & ((unsigned char) 0x3F))); if (unipoint > 0xFFFF) { unipoint = (unsigned int) '?'; } @@ -2225,7 +2215,7 @@ static WCHAR *utf8_to_utf16(unsigned char *bytes) static int put_utf8(WCHAR ch, unsigned char *target, int sz, int *pos) { - Uint x = (Uint) ch; + unsigned int x = (unsigned int) ch; if (x < 0x80) { if (*pos >= sz) { return -1; diff --git a/erts/etc/common/escript.c b/erts/etc/common/escript.c index 7f0af77a4c..8241675200 100644 --- a/erts/etc/common/escript.c +++ b/erts/etc/common/escript.c @@ -20,16 +20,8 @@ /* * Purpose: escript front-end. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#ifdef __WIN32__ -#include <winbase.h> -#endif - -#include <ctype.h> +#include "etc_common.h" static int debug = 0; /* Bit flags for debug printouts. */ @@ -74,7 +66,6 @@ static void error(char* format, ...); static void* emalloc(size_t size); static void efree(void *p); static char* strsave(char* string); -static void push_words(char* src); static int run_erlang(char* name, char** argv); static char* get_default_emulator(char* progname); #ifdef __WIN32__ @@ -583,26 +574,6 @@ main(int argc, char** argv) return run_erlang(eargv[0], eargv); } -static void -push_words(char* src) -{ - char sbuf[PMAX]; - char* dst; - - dst = sbuf; - while ((*dst++ = *src++) != '\0') { - if (isspace((int)*src)) { - *dst = '\0'; - PUSH(strsave(sbuf)); - dst = sbuf; - do { - src++; - } while (isspace((int)*src)); - } - } - if (sbuf[0]) - PUSH(strsave(sbuf)); -} #ifdef __WIN32__ wchar_t *make_commandline(char **argv) { diff --git a/erts/etc/common/etc_common.h b/erts/etc/common/etc_common.h new file mode 100644 index 0000000000..3f26064a9e --- /dev/null +++ b/erts/etc/common/etc_common.h @@ -0,0 +1,65 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2017. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * Purpose: common includes for all etc programs + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#if !defined(__WIN32__) +# include <dirent.h> +# include <limits.h> +# include <sys/stat.h> +# include <sys/types.h> +# include <unistd.h> +#else +# include <windows.h> +# include <io.h> +# include <winbase.h> +# include <process.h> +#endif + +#include <errno.h> +#include <fcntl.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> + +/* + * Make sure that MAXPATHLEN is defined. + */ +#ifndef MAXPATHLEN +# ifdef PATH_MAX +# define MAXPATHLEN PATH_MAX +# else +# define MAXPATHLEN 2048 +# endif +#endif + +#include "erl_printf.h" + +#ifdef __WIN32__ +/* FIXE ME config_win32.h? */ +#define HAVE_STRERROR 1 +#define snprintf _snprintf +#endif diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c index 6bae9f96b7..b64cbb4a92 100644 --- a/erts/etc/common/typer.c +++ b/erts/etc/common/typer.c @@ -20,16 +20,8 @@ /* * Purpose: Typer front-end. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#ifdef __WIN32__ -#include <winbase.h> -#endif -#include <ctype.h> +#include "etc_common.h" #define NO 0 #define YES 1 diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 30f2d831b5..8d5882cf32 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -139,29 +139,6 @@ while [ $# -gt 0 ]; do shift unset DISPLAY ;; - "-smp") - shift - if [ $# -le 0 ]; then - eeargs_add -smp - else - case $1 in - disable) - shift - eeargs_add -smpdisable - ;; - enable) - shift - eeargs_add -smp - ;; - *) - eeargs_add -smp - esac - fi - ;; - "-smpdisable") - shift - eeargs_add -smpdisable - ;; "-lcnt") shift cargs="$cargs -lcnt" diff --git a/erts/etc/unix/dyn_erl.c b/erts/etc/unix/dyn_erl.c index d6d2201648..5c7c3cad38 100644 --- a/erts/etc/unix/dyn_erl.c +++ b/erts/etc/unix/dyn_erl.c @@ -22,13 +22,7 @@ * This is a C version of the erl Bourne shell script */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include <stdlib.h> -#include <stdarg.h> +#include "etc_common.h" #define BOOL int #define TRUE 1 diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 8f70f879d5..95e065b156 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -2316,40 +2316,46 @@ end define etp-port-sched-flags-int # Args: int # - if ($arg0 & 0x1) + if ($arg0 & (1 << 0)) printf " in-run-queue" end - if ($arg0 & 0x2) + if ($arg0 & (1 << 1)) printf " executing" end - if ($arg0 & 0x4) + if ($arg0 & (1 << 2)) printf " have-tasks" end - if ($arg0 & 0x8) + if ($arg0 & (1 << 3)) printf " exited" end - if ($arg0 & 0x10) + if ($arg0 & (1 << 4)) printf " busy-port" end - if ($arg0 & 0x20) + if ($arg0 & (1 << 5)) printf " busy-port-q" end - if ($arg0 & 0x40) + if ($arg0 & (1 << 6)) printf " chk-unset-busy-port-q" end - if ($arg0 & 0x80) + if ($arg0 & (1 << 7)) printf " have-busy-tasks" end - if ($arg0 & 0x100) + if ($arg0 & (1 << 8)) printf " have-nosuspend-tasks" end - if ($arg0 & 0x200) + if ($arg0 & (1 << 9)) printf " parallelism" end - if ($arg0 & 0x400) + if ($arg0 & (1 << 10)) printf " force-sched" end - if ($arg0 & 0xfffff800) + if ($arg0 & (1 << 11)) + printf " exiting" + end + if ($arg0 & (1 << 12)) + printf " exec-imm" + end + if ($arg0 & 0xffffc000) printf " GARBAGE" end printf "\n" @@ -2850,6 +2856,14 @@ define etp-run-queue-info-internal printf " Pointer: (ErtsRunQueue *) %p\n", $runq end +define etp-fds + if $_exitsignal == -1 + call erts_check_io_debug(0) + else + printf "Not yet implemented for core files" + end +end + define etp-disasm-1 set $code_ptr = ((BeamInstr*)$arg0) set $addr = *$code_ptr @@ -4314,6 +4328,10 @@ document etp-init %--------------------------------------------------------------------------- end +define hook-run + set $_exitsignal = -1 +end + etp-init help etp-init etp-show diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index f05c729eeb..f928163705 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -627,12 +627,14 @@ static void pass_on(pid_t childpid) status("Pty master read; "); #endif if ((len = sf_read(mfd, buf, BUFSIZ)) <= 0) { + int saved_errno = errno; sf_close(rfd); if(wfd) sf_close(wfd); sf_close(mfd); unlink(fifo1); unlink(fifo2); if (len < 0) { + errno = saved_errno; if(errno == EIO) ERROR0(LOG_ERR,"Erlang closed the connection."); else @@ -1342,13 +1344,15 @@ static int sf_open(const char *path, int type, mode_t mode) { return fd; } + static int sf_close(int fd) { int res = 0; - do { res = close(fd); } while(fd < 0 && errno == EINTR); + do { res = close(fd); } while(res < 0 && errno == EINTR); return res; } + /* Extract any control sequences that are ment only for run_erl * and should not be forwarded to the pty. */ diff --git a/erts/lib_src/common/erl_printf.c b/erts/lib_src/common/erl_printf.c index 7781fc2196..9031a4c5b7 100644 --- a/erts/lib_src/common/erl_printf.c +++ b/erts/lib_src/common/erl_printf.c @@ -63,7 +63,7 @@ void (*erts_printf_unblock_fpe)(int) = NULL; #undef FWRITE #undef PUTC_ON_SMALL_WRITES -#if defined(USE_THREADS) && defined(HAVE_FLOCKFILE) +#if defined(HAVE_FLOCKFILE) # define FLOCKFILE(FP) flockfile(FP) # define FUNLOCKFILE(FP) funlockfile(FP) # ifdef HAVE_PUTC_UNLOCKED @@ -73,11 +73,7 @@ void (*erts_printf_unblock_fpe)(int) = NULL; # ifdef HAVE_FWRITE_UNLOCKED # define FWRITE fwrite_unlocked # endif -#endif -#if !defined(USE_THREADS) && defined(putc) && !defined(fwrite) -# define PUTC_ON_SMALL_WRITES -#endif -#if !defined(FLOCKFILE) || !defined(FUNLOCKFILE) +#else # define FLOCKFILE(FP) # define FUNLOCKFILE(FP) #endif diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 58c17dc416..452f3dcc07 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 6691749dcb..5416826f19 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 2acb1f1211..1c8d0e626a 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex a959ebaaf2..5048bdb846 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 72dd804412..c6c105d36a 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -48,6 +48,12 @@ await_sched_wall_time_modifications/2, gather_gc_info_result/1]). +-export([dist_ctrl_input_handler/2, + dist_ctrl_put_data/2, + dist_ctrl_get_data/1, + dist_ctrl_get_data_notification/1, + dist_get_stat/1]). + -deprecated([now/0]). %% Get rid of autoimports of spawn to avoid clashes with ourselves. @@ -87,6 +93,14 @@ -export_type([prepared_code/0]). +-opaque dist_handle() :: atom(). + +-export_type([dist_handle/0]). + +-type iovec() :: [binary()]. + +-export_type([iovec/0]). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Native code BIF stubs and their types %% (BIF's actually implemented in this module goes last in the file) @@ -124,7 +138,7 @@ has_prepared_code_on_load/1, hibernate/3]). -export([insert_element/3]). -export([integer_to_binary/1, integer_to_list/1]). --export([iolist_size/1, iolist_to_binary/1]). +-export([iolist_size/1, iolist_to_binary/1, iolist_to_iovec/1]). -export([is_alive/0, is_builtin/3, is_process_alive/1, length/1, link/1]). -export([list_to_atom/1, list_to_binary/1]). -export([list_to_bitstring/1, list_to_existing_atom/1, list_to_float/1]). @@ -1079,6 +1093,12 @@ iolist_size(_Item) -> iolist_to_binary(_IoListOrBinary) -> erlang:nif_error(undefined). +%% iolist_to_iovec/1 +-spec erlang:iolist_to_iovec(IoListOrBinary) -> iovec() when + IoListOrBinary :: iolist() | binary(). +iolist_to_iovec(_IoListOrBinary) -> + erlang:nif_error(undefined). + %% is_alive/0 -spec is_alive() -> boolean(). is_alive() -> @@ -1641,7 +1661,7 @@ setnode(_P1, _P2) -> erlang:nif_error(undefined). %% setnode/3 --spec erlang:setnode(P1, P2, P3) -> true when +-spec erlang:setnode(P1, P2, P3) -> dist_handle() when P1 :: atom(), P2 :: port(), P3 :: {term(), term(), term(), term()}. @@ -2324,7 +2344,8 @@ spawn_opt(_Tuple) -> MSAcc_Thread :: #{ type := MSAcc_Thread_Type, id := MSAcc_Thread_Id, counters := MSAcc_Counters}, - MSAcc_Thread_Type :: scheduler | async | aux, + MSAcc_Thread_Type :: async | aux | dirty_io_scheduler + | dirty_cpu_scheduler | poll | scheduler, MSAcc_Thread_Id :: non_neg_integer(), MSAcc_Counters :: #{ MSAcc_Thread_State => non_neg_integer() }, MSAcc_Thread_State :: alloc | aux | bif | busy_wait | check_io | @@ -3204,6 +3225,47 @@ port_get_data(_Port) -> erlang:nif_error(undefined). %% +%% Distribution channel management +%% + +-spec erlang:dist_ctrl_input_handler(DHandle, InputHandler) -> 'ok' when + DHandle :: dist_handle(), + InputHandler :: pid(). + +dist_ctrl_input_handler(_DHandle, _InputHandler) -> + erlang:nif_error(undefined). + +-spec erlang:dist_ctrl_put_data(DHandle, Data) -> 'ok' when + DHandle :: dist_handle(), + Data :: iodata(). + +dist_ctrl_put_data(_DHandle, _Data) -> + erlang:nif_error(undefined). + +-spec erlang:dist_ctrl_get_data(DHandle) -> Data | 'none' when + DHandle :: dist_handle(), + Data :: iodata(). + +dist_ctrl_get_data(_DHandle) -> + erlang:nif_error(undefined). + +-spec erlang:dist_ctrl_get_data_notification(DHandle) -> 'ok' when + DHandle :: dist_handle(). + +dist_ctrl_get_data_notification(_DHandle) -> + erlang:nif_error(undefined). + +-spec erlang:dist_get_stat(DHandle) -> Res when + DHandle :: dist_handle(), + InputPackets :: non_neg_integer(), + OutputPackets :: non_neg_integer(), + PendingOutputPackets :: boolean(), + Res :: {'ok', InputPackets, OutputPackets, PendingOutputPackets}. + +dist_get_stat(_DHandle) -> + erlang:nif_error(undefined). + +%% %% If the emulator wants to perform a distributed command and %% a connection is not established to the actual node the following %% functions are called in order to set up the connection and then diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 7ab06164b4..beb29a7c89 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -37,7 +37,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0.1"]} + {runtime_dependencies, ["stdlib-3.4.3", "kernel-5.4.1", "sasl-3.0.1"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 26fb1458af..bb1824ecd4 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -61,6 +61,8 @@ -export([trace/3, trace_pattern/3]). +-export([dist_ctrl_put_data/2]). + %% Auto import name clash -export([check_process_code/1]). @@ -461,3 +463,38 @@ trace(_PidSpec, _How, _FlagList) -> FlagList :: [ ]. trace_pattern(_MFA, _MatchSpec, _FlagList) -> erlang:nif_error(undefined). + +-spec dist_ctrl_put_data(DHandle, Data) -> 'ok' when + DHandle :: erlang:dist_handle(), + Data :: iolist(). + +dist_ctrl_put_data(DHandle, IoList) -> + %% + %% Helper for erlang:dist_ctrl_put_data/2 + %% + %% erlang:dist_ctrl_put_data/2 traps to + %% this function if second argument is + %% a list... + %% + try + Binary = erlang:iolist_to_binary(IoList), + %% Restart erlang:dist_ctrl_put_data/2 + %% with the iolist converted to a binary... + erlang:dist_ctrl_put_data(DHandle, Binary) + catch + Class : Reason -> + %% Throw exception as if thrown from + %% erlang:dist_ctrl_put_data/2 ... + RootST = try erlang:error(Reason) + catch + error:Reason -> + case erlang:get_stacktrace() of + [] -> []; + ST -> tl(ST) + end + end, + StackTrace = [{erlang, dist_ctrl_put_data, + [DHandle, IoList], []} + | RootST], + erlang:raise(Class, Reason, StackTrace) + end. diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 1ccf8d599f..34a9f6b8b9 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -200,6 +200,8 @@ boot(BootArgs) -> register(init, self()), process_flag(trap_exit, true), + %% Load the zlib nif + zlib:on_load(), %% Load the tracer nif erl_tracer:on_load(), diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl index 8cd3e39fd7..3170ab6351 100644 --- a/erts/preloaded/src/zlib.erl +++ b/erts/preloaded/src/zlib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. +%% Copyright Ericsson AB 2003-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,19 +21,29 @@ -module(zlib). -export([open/0,close/1,deflateInit/1,deflateInit/2,deflateInit/6, - deflateSetDictionary/2,deflateReset/1,deflateParams/3, - deflate/2,deflate/3,deflateEnd/1, - inflateInit/1,inflateInit/2, - inflateSetDictionary/2,inflateGetDictionary/1, - inflateSync/1,inflateReset/1,inflate/2,inflateEnd/1, - inflateChunk/1, inflateChunk/2, - setBufSize/2,getBufSize/1, - crc32/1,crc32/2,crc32/3,adler32/2,adler32/3,getQSize/1, - crc32_combine/4,adler32_combine/4, - compress/1,uncompress/1,zip/1,unzip/1, - gzip/1,gunzip/1]). - --export_type([zstream/0, zlevel/0, zwindowbits/0, zmemlevel/0, zstrategy/0]). + deflateSetDictionary/2,deflateReset/1,deflateParams/3, + deflate/2,deflate/3,deflateEnd/1, + inflateInit/1,inflateInit/2,inflateInit/3, + inflateSetDictionary/2,inflateGetDictionary/1, inflateReset/1, + inflate/2,inflate/3,inflateEnd/1, + inflateChunk/2,inflateChunk/1, + safeInflate/2, + setBufSize/2,getBufSize/1, + crc32/1,crc32/2,crc32/3,adler32/2,adler32/3, + crc32_combine/4,adler32_combine/4, + compress/1,uncompress/1,zip/1,unzip/1, + gzip/1,gunzip/1]). + +-export([on_load/0]). + +%% These are soft-deprecated until OTP 21. +% -deprecated([inflateChunk/1, inflateChunk/2, +% getBufSize/1, setBufSize/2, +% crc32/1,crc32/2,crc32/3,adler32/2,adler32/3, +% crc32_combine/4,adler32_combine/4]). + +-export_type([zstream/0, zflush/0, zlevel/0, zwindowbits/0, zmemlevel/0, + zstrategy/0]). %% flush argument encoding -define(Z_NO_FLUSH, 0). @@ -56,116 +66,80 @@ %% deflate compression method -define(Z_DEFLATED, 8). --define(Z_NULL, 0). - -define(MAX_WBITS, 15). -%% gzip defs (rfc 1952) - --define(ID1, 16#1f). --define(ID2, 16#8b). - --define(FTEXT, 16#01). --define(FHCRC, 16#02). --define(FEXTRA, 16#04). --define(FNAME, 16#08). --define(FCOMMENT, 16#10). --define(RESERVED, 16#E0). - --define(OS_MDDOS, 0). --define(OS_AMIGA, 1). --define(OS_OPENVMS, 2). --define(OS_UNIX, 3). --define(OS_VMCMS, 4). --define(OS_ATARI, 5). --define(OS_OS2, 6). --define(OS_MAC, 7). --define(OS_ZSYS, 8). --define(OS_CPM, 9). --define(OS_TOP20, 10). --define(OS_NTFS, 11). --define(OS_QDOS, 12). --define(OS_ACORN, 13). --define(OS_UNKNOWN,255). - --define(DEFLATE_INIT, 1). --define(DEFLATE_INIT2, 2). --define(DEFLATE_SETDICT, 3). --define(DEFLATE_RESET, 4). --define(DEFLATE_END, 5). --define(DEFLATE_PARAMS, 6). --define(DEFLATE, 7). - --define(INFLATE_INIT, 8). --define(INFLATE_INIT2, 9). --define(INFLATE_SETDICT, 10). --define(INFLATE_GETDICT, 11). --define(INFLATE_SYNC, 12). --define(INFLATE_RESET, 13). --define(INFLATE_END, 14). --define(INFLATE, 15). --define(INFLATE_CHUNK, 26). - --define(CRC32_0, 16). --define(CRC32_1, 17). --define(CRC32_2, 18). - --define(SET_BUFSZ, 19). --define(GET_BUFSZ, 20). --define(GET_QSIZE, 21). - --define(ADLER32_1, 22). --define(ADLER32_2, 23). - --define(CRC32_COMBINE, 24). --define(ADLER32_COMBINE, 25). +-define(DEFAULT_MEMLEVEL, 8). +-define(DEFAULT_WBITS, 15). + +-define(EOS_BEHAVIOR_ERROR, 0). +-define(EOS_BEHAVIOR_RESET, 1). +-define(EOS_BEHAVIOR_CUT, 2). + +%% Chunk sizes are hardcoded on account of them screwing with the +%% predictability of the system. zlib is incapable of trapping so we need to +%% ensure that it never operates on any significant amount of data. +-define(DEFLATE_IN_CHUNKSIZE, 8 bsl 10). +-define(DEFLATE_OUT_CHUNKSIZE, 8 bsl 10). +-define(INFLATE_IN_CHUNKSIZE, 8 bsl 10). +-define(INFLATE_OUT_CHUNKSIZE, 16 bsl 10). %%------------------------------------------------------------------------ -%% Main data types of the file --type zstream() :: port(). +%% Public data types. +-type zstream() :: term(). +-type zflush() :: 'none' | 'sync' | 'full' | 'finish'. -%% Auxiliary data types of the file --type zlevel() :: 'none' | 'default' | 'best_compression' | 'best_speed' - | 0..9. --type zmethod() :: 'deflated'. +-type zlevel() :: + 'none' | 'default' | 'best_compression' | 'best_speed' | 0..9. +-type zstrategy() :: 'default' | 'filtered' | 'huffman_only' | 'rle'. + +-type zmemlevel() :: 1..9. -type zwindowbits() :: -15..-8 | 8..47. --type zmemlevel() :: 1..9. --type zstrategy() :: 'default' | 'filtered' | 'huffman_only' | 'rle'. + +%% Private data types. + +-type zmethod() :: 'deflated'. + +-record(zlib_opts, { + stream :: zstream(), + method :: term(), + input_chunk_size :: integer(), + output_chunk_size :: integer(), + flush :: integer() + }). %%------------------------------------------------------------------------ -%% open a z_stream +on_load() -> + case erlang:load_nif(atom_to_list(?MODULE), 0) of + ok -> ok + end. + -spec open() -> zstream(). open() -> - open_port({spawn, "zlib_drv"}, [binary]). + open_nif(). +open_nif() -> + erlang:nif_error(undef). -%% close and release z_stream -spec close(Z) -> 'ok' when Z :: zstream(). close(Z) -> - try - true = port_close(Z), - receive %In case the caller is the owner and traps exits - {'EXIT',Z,_} -> ok - after 0 -> ok - end - catch _:_ -> erlang:error(badarg) - end. + close_nif(Z). +close_nif(_Z) -> + erlang:nif_error(undef). -spec deflateInit(Z) -> 'ok' when Z :: zstream(). deflateInit(Z) -> - call(Z, ?DEFLATE_INIT, <<?Z_DEFAULT_COMPRESSION:32>>). + deflateInit(Z, default). -spec deflateInit(Z, Level) -> 'ok' when Z :: zstream(), Level :: zlevel(). deflateInit(Z, Level) -> - call(Z, ?DEFLATE_INIT, <<(arg_level(Level)):32>>). + deflateInit(Z, Level, deflated, ?DEFAULT_WBITS, ?DEFAULT_MEMLEVEL, default). --spec deflateInit(Z, Level, Method, - WindowBits, MemLevel, Strategy) -> 'ok' when +-spec deflateInit(Z, Level, Method, WindowBits, MemLevel, Strategy) -> 'ok' when Z :: zstream(), Level :: zlevel(), Method :: zmethod(), @@ -173,31 +147,49 @@ deflateInit(Z, Level) -> MemLevel :: zmemlevel(), Strategy :: zstrategy(). deflateInit(Z, Level, Method, WindowBits, MemLevel, Strategy) -> - call(Z, ?DEFLATE_INIT2, <<(arg_level(Level)):32, - (arg_method(Method)):32, - (arg_bitsz(WindowBits)):32, - (arg_mem(MemLevel)):32, - (arg_strategy(Strategy)):32>>). + deflateInit_nif(Z, + arg_level(Level), + arg_method(Method), + arg_bitsz(WindowBits), + arg_mem(MemLevel), + arg_strategy(Strategy)). +deflateInit_nif(_Z, _Level, _Method, _WindowBits, _MemLevel, _Strategy) -> + erlang:nif_error(undef). -spec deflateSetDictionary(Z, Dictionary) -> Adler32 when Z :: zstream(), Dictionary :: iodata(), Adler32 :: integer(). deflateSetDictionary(Z, Dictionary) -> - call(Z, ?DEFLATE_SETDICT, Dictionary). + deflateSetDictionary_nif(Z, Dictionary). +deflateSetDictionary_nif(_Z, _Dictionary) -> + erlang:nif_error(undef). -spec deflateReset(Z) -> 'ok' when Z :: zstream(). deflateReset(Z) -> - call(Z, ?DEFLATE_RESET, []). + deflateReset_nif(Z). +deflateReset_nif(_Z) -> + erlang:nif_error(undef). -spec deflateParams(Z, Level, Strategy) -> ok when Z :: zstream(), Level :: zlevel(), Strategy :: zstrategy(). -deflateParams(Z, Level, Strategy) -> - call(Z, ?DEFLATE_PARAMS, <<(arg_level(Level)):32, - (arg_strategy(Strategy)):32>>). +deflateParams(Z, Level0, Strategy0) -> + Level = arg_level(Level0), + Strategy = arg_strategy(Strategy0), + case deflateParams_nif(Z, Level, Strategy) of + buf_error -> + %% We had data left in the pipe; flush everything and stash it away + %% for the next deflate call before trying again. + Output = deflate(Z, <<>>, full), + save_progress(Z, deflate, Output), + deflateParams_nif(Z, Level, Strategy); + Any -> Any + end. +deflateParams_nif(_Z, _Level, _Strategy) -> + erlang:nif_error(undef). -spec deflate(Z, Data) -> Compressed when Z :: zstream(), @@ -209,170 +201,239 @@ deflate(Z, Data) -> -spec deflate(Z, Data, Flush) -> Compressed when Z :: zstream(), Data :: iodata(), - Flush :: none | sync | full | finish, + Flush :: zflush(), Compressed :: iolist(). deflate(Z, Data, Flush) -> - try port_command(Z, Data) of - true -> - _ = call(Z, ?DEFLATE, <<(arg_flush(Flush)):32>>), - collect(Z) - catch - error:_Err -> - flush(Z), - erlang:error(badarg) - end. + Progress = restore_progress(Z, deflate), + enqueue_input(Z, Data), + append_iolist(Progress, dequeue_all_chunks(Z, deflate_opts(Flush))). + +deflate_opts(Flush) -> + #zlib_opts{ + method = fun deflate_nif/4, + input_chunk_size = ?DEFLATE_IN_CHUNKSIZE, + output_chunk_size = ?DEFLATE_OUT_CHUNKSIZE, + flush = arg_flush(Flush) + }. + +deflate_nif(_Z, _InputChSize, _OutputChSize, _Flush) -> + erlang:nif_error(undef). -spec deflateEnd(Z) -> 'ok' when Z :: zstream(). deflateEnd(Z) -> - call(Z, ?DEFLATE_END, []). + deflateEnd_nif(Z). +deflateEnd_nif(_Z) -> + erlang:nif_error(undef). -spec inflateInit(Z) -> 'ok' when Z :: zstream(). inflateInit(Z) -> - call(Z, ?INFLATE_INIT, []). + inflateInit(Z, ?DEFAULT_WBITS). -spec inflateInit(Z, WindowBits) -> 'ok' when Z :: zstream(), WindowBits :: zwindowbits(). -inflateInit(Z, WindowBits) -> - call(Z, ?INFLATE_INIT2, <<(arg_bitsz(WindowBits)):32>>). +inflateInit(Z, WindowBits) -> + inflateInit(Z, WindowBits, cut). + +-spec inflateInit(Z, WindowBits, EoSBehavior) -> 'ok' when + Z :: zstream(), + WindowBits :: zwindowbits(), + EoSBehavior :: error | reset | cut. +inflateInit(Z, WindowBits, EoSBehavior) -> + inflateInit_nif(Z, arg_bitsz(WindowBits), arg_eos_behavior(EoSBehavior)). +inflateInit_nif(_Z, _WindowBits, _EoSBehavior) -> + erlang:nif_error(undef). -spec inflateSetDictionary(Z, Dictionary) -> 'ok' when Z :: zstream(), Dictionary :: iodata(). -inflateSetDictionary(Z, Dictionary) -> - call(Z, ?INFLATE_SETDICT, Dictionary). +inflateSetDictionary(Z, Dictionary) -> + inflateSetDictionary_nif(Z, Dictionary). +inflateSetDictionary_nif(_Z, _Dictionary) -> + erlang:nif_error(undef). -spec inflateGetDictionary(Z) -> Dictionary when Z :: zstream(), - Dictionary :: iolist(). + Dictionary :: binary(). inflateGetDictionary(Z) -> - _ = call(Z, ?INFLATE_GETDICT, []), - collect(Z). - --spec inflateSync(zstream()) -> 'ok'. -inflateSync(Z) -> - call(Z, ?INFLATE_SYNC, []). + case inflateGetDictionary_nif(Z) of + Dictionary when is_binary(Dictionary) -> + Dictionary; + not_supported -> + erlang:error(enotsup) + end. +inflateGetDictionary_nif(_Z) -> + erlang:nif_error(undef). -spec inflateReset(Z) -> 'ok' when Z :: zstream(). -inflateReset(Z) -> - call(Z, ?INFLATE_RESET, []). +inflateReset(Z) -> + inflateReset_nif(Z). +inflateReset_nif(_Z) -> + erlang:nif_error(undef). -spec inflate(Z, Data) -> Decompressed when Z :: zstream(), Data :: iodata(), Decompressed :: iolist(). inflate(Z, Data) -> - try port_command(Z, Data) of - true -> - _ = call(Z, ?INFLATE, <<?Z_NO_FLUSH:32>>), - collect(Z) - catch - error:_Err -> - flush(Z), - erlang:error(badarg) + inflate(Z, Data, []). + +-spec inflate(Z, Data, Options) -> Decompressed when + Z :: zstream(), + Data :: iodata(), + Options :: list({exception_on_need_dict, boolean()}), + Decompressed :: iolist() | + {need_dictionary, + Adler32 :: integer(), + Output :: iolist()}. +inflate(Z, Data, Options) -> + enqueue_input(Z, Data), + Result = dequeue_all_chunks(Z, inflate_opts()), + case proplist_get_value(Options, exception_on_need_dict, true) of + true -> exception_on_need_dict(Z, Result); + false -> Result end. +inflate_nif(_Z, _InputChSize, _OutputChSize, _Flush) -> + erlang:nif_error(undef). + +inflate_opts() -> + #zlib_opts{ + method = fun inflate_nif/4, + input_chunk_size = ?INFLATE_IN_CHUNKSIZE, + output_chunk_size = ?INFLATE_OUT_CHUNKSIZE, + flush = arg_flush(none) + }. + -spec inflateChunk(Z, Data) -> Decompressed | {more, Decompressed} when Z :: zstream(), Data :: iodata(), Decompressed :: iolist(). inflateChunk(Z, Data) -> - try port_command(Z, Data) of - true -> - inflateChunk(Z) - catch - error:_Err -> - flush(Z), - erlang:error(badarg) - end. + enqueue_input(Z, Data), + inflateChunk(Z). -spec inflateChunk(Z) -> Decompressed | {more, Decompressed} when Z :: zstream(), Decompressed :: iolist(). inflateChunk(Z) -> - Status = call(Z, ?INFLATE_CHUNK, []), - Data = receive - {Z, {data, Bin}} -> - Bin - after 0 -> - [] - end, - - case Status of - Good when (Good == ok) orelse (Good == stream_end) -> - Data; - inflate_has_more -> - {more, Data} - end. + Opts0 = inflate_opts(), + Opts = Opts0#zlib_opts { output_chunk_size = getBufSize(Z) }, + + Result0 = dequeue_next_chunk(Z, Opts), + Result1 = exception_on_need_dict(Z, Result0), + yield_inflateChunk(Z, Result1). + +yield_inflateChunk(_Z, {continue, Output}) -> + {more, lists:flatten(Output)}; +yield_inflateChunk(_Z, {finished, Output}) -> + lists:flatten(Output). + +exception_on_need_dict(Z, {need_dictionary, Adler, Output}) -> + Progress = restore_progress(Z, inflate), + save_progress(Z, inflate, append_iolist(Progress, Output)), + erlang:error({need_dictionary, Adler}); +exception_on_need_dict(Z, {Mark, Output}) -> + Progress = restore_progress(Z, inflate), + {Mark, append_iolist(Progress, Output)}; +exception_on_need_dict(Z, Output) when is_list(Output); is_binary(Output) -> + Progress = restore_progress(Z, inflate), + append_iolist(Progress, Output). + +-spec safeInflate(Z, Data) -> Result when + Z :: zstream(), + Data :: iodata(), + Result :: {continue, Output :: iolist()} | + {finished, Output :: iolist()} | + {need_dictionary, + Adler32 :: integer(), + Output :: iolist()}. +safeInflate(Z, Data) -> + enqueue_input(Z, Data), + dequeue_next_chunk(Z, inflate_opts()). -spec inflateEnd(Z) -> 'ok' when Z :: zstream(). inflateEnd(Z) -> - call(Z, ?INFLATE_END, []). + inflateEnd_nif(Z). +inflateEnd_nif(_Z) -> + erlang:nif_error(undef). -spec setBufSize(Z, Size) -> 'ok' when Z :: zstream(), Size :: non_neg_integer(). -setBufSize(Z, Size) -> - call(Z, ?SET_BUFSZ, <<Size:32>>). +setBufSize(Z, Size) when is_integer(Size), Size > 16, Size < (1 bsl 24) -> + setBufSize_nif(Z, Size); +setBufSize(_Z, _Size) -> + erlang:error(badarg). +setBufSize_nif(_Z, _Size) -> + erlang:nif_error(undef). --spec getBufSize(Z) -> Size when - Z :: zstream(), - Size :: non_neg_integer(). +-spec getBufSize(Z) -> non_neg_integer() when + Z :: zstream(). getBufSize(Z) -> - call(Z, ?GET_BUFSZ, []). + getBufSize_nif(Z). +getBufSize_nif(_Z) -> + erlang:nif_error(undef). -spec crc32(Z) -> CRC when Z :: zstream(), CRC :: integer(). crc32(Z) -> - call(Z, ?CRC32_0, []). + crc32_nif(Z). +crc32_nif(_Z) -> + erlang:nif_error(undef). -spec crc32(Z, Data) -> CRC when Z :: zstream(), Data :: iodata(), CRC :: integer(). -crc32(Z, Data) -> - call(Z, ?CRC32_1, Data). +crc32(Z, Data) when is_reference(Z) -> + erlang:crc32(Data); +crc32(_Z, _Data) -> + erlang:error(badarg). -spec crc32(Z, PrevCRC, Data) -> CRC when Z :: zstream(), PrevCRC :: integer(), Data :: iodata(), CRC :: integer(). -crc32(Z, CRC, Data) -> - call(Z, ?CRC32_2, [<<CRC:32>>, Data]). +crc32(Z, CRC, Data) when is_reference(Z) -> + erlang:crc32(CRC, Data); +crc32(_Z, _CRC, _Data) -> + erlang:error(badarg). + +-spec crc32_combine(Z, CRC1, CRC2, Size2) -> CRC when + Z :: zstream(), + CRC :: integer(), + CRC1 :: integer(), + CRC2 :: integer(), + Size2 :: integer(). +crc32_combine(Z, CRC1, CRC2, Size2) when is_reference(Z) -> + erlang:crc32_combine(CRC1, CRC2, Size2); +crc32_combine(_Z, _CRC1, _CRC2, _Size2) -> + erlang:error(badarg). -spec adler32(Z, Data) -> CheckSum when Z :: zstream(), Data :: iodata(), CheckSum :: integer(). -adler32(Z, Data) -> - call(Z, ?ADLER32_1, Data). +adler32(Z, Data) when is_reference(Z) -> + erlang:adler32(Data); +adler32(_Z, _Data) -> + erlang:error(badarg). -spec adler32(Z, PrevAdler, Data) -> CheckSum when Z :: zstream(), PrevAdler :: integer(), Data :: iodata(), CheckSum :: integer(). -adler32(Z, Adler, Data) when is_integer(Adler) -> - call(Z, ?ADLER32_2, [<<Adler:32>>, Data]); -adler32(_Z, _Adler, _Data) -> - erlang:error(badarg). - --spec crc32_combine(Z, CRC1, CRC2, Size2) -> CRC when - Z :: zstream(), - CRC :: integer(), - CRC1 :: integer(), - CRC2 :: integer(), - Size2 :: integer(). -crc32_combine(Z, CRC1, CRC2, Len2) - when is_integer(CRC1), is_integer(CRC2), is_integer(Len2) -> - call(Z, ?CRC32_COMBINE, <<CRC1:32, CRC2:32, Len2:32>>); -crc32_combine(_Z, _CRC1, _CRC2, _Len2) -> +adler32(Z, Adler, Data) when is_reference(Z) -> + erlang:adler32(Adler, Data); +adler32(_Z, _Adler, _Data) -> erlang:error(badarg). -spec adler32_combine(Z, Adler1, Adler2, Size2) -> Adler when @@ -381,16 +442,11 @@ crc32_combine(_Z, _CRC1, _CRC2, _Len2) -> Adler1 :: integer(), Adler2 :: integer(), Size2 :: integer(). -adler32_combine(Z, Adler1, Adler2, Len2) - when is_integer(Adler1), is_integer(Adler2), is_integer(Len2) -> - call(Z, ?ADLER32_COMBINE, <<Adler1:32, Adler2:32, Len2:32>>); -adler32_combine(_Z, _Adler1, _Adler2, _Len2) -> +adler32_combine(Z, Adler1, Adler2, Size2) when is_reference(Z) -> + erlang:adler32_combine(Adler1, Adler2, Size2); +adler32_combine(_Z, _Adler1, _Adler2, _Size2) -> erlang:error(badarg). --spec getQSize(zstream()) -> non_neg_integer(). -getQSize(Z) -> - call(Z, ?GET_QSIZE, []). - %% compress/uncompress zlib with header -spec compress(Data) -> Compressed when Data :: iodata(), @@ -398,13 +454,13 @@ getQSize(Z) -> compress(Data) -> Z = open(), Bs = try - deflateInit(Z, default), - B = deflate(Z, Data, finish), - deflateEnd(Z), - B - after - close(Z) - end, + deflateInit(Z, default), + B = deflate(Z, Data, finish), + deflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs). -spec uncompress(Data) -> Decompressed when @@ -416,14 +472,14 @@ uncompress(Data) -> if Size >= 8 -> Z = open(), - Bs = try - inflateInit(Z), - B = inflate(Z, Data), - inflateEnd(Z), - B - after - close(Z) - end, + Bs = try + inflateInit(Z), + B = inflate(Z, Data), + inflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs); true -> erlang:error(data_error) @@ -440,13 +496,13 @@ uncompress(Data) -> zip(Data) -> Z = open(), Bs = try - deflateInit(Z, default, deflated, -?MAX_WBITS, 8, default), - B = deflate(Z, Data, finish), - deflateEnd(Z), - B - after - close(Z) - end, + deflateInit(Z, default, deflated, -?MAX_WBITS, 8, default), + B = deflate(Z, Data, finish), + deflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs). -spec unzip(Data) -> Decompressed when @@ -455,28 +511,28 @@ zip(Data) -> unzip(Data) -> Z = open(), Bs = try - inflateInit(Z, -?MAX_WBITS), - B = inflate(Z, Data), - inflateEnd(Z), - B - after - close(Z) - end, + inflateInit(Z, -?MAX_WBITS), + B = inflate(Z, Data), + inflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs). - + -spec gzip(Data) -> Compressed when Data :: iodata(), Compressed :: binary(). gzip(Data) -> Z = open(), Bs = try - deflateInit(Z, default, deflated, 16+?MAX_WBITS, 8, default), - B = deflate(Z, Data, finish), - deflateEnd(Z), - B - after - close(Z) - end, + deflateInit(Z, default, deflated, 16+?MAX_WBITS, 8, default), + B = deflate(Z, Data, finish), + deflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs). -spec gunzip(Data) -> Decompressed when @@ -485,92 +541,155 @@ gzip(Data) -> gunzip(Data) -> Z = open(), Bs = try - inflateInit(Z, 16+?MAX_WBITS), - B = inflate(Z, Data), - inflateEnd(Z), - B - after - close(Z) - end, + inflateInit(Z, 16+?MAX_WBITS, reset), + B = inflate(Z, Data), + inflateEnd(Z), + B + after + close(Z) + end, iolist_to_binary(Bs). --spec collect(zstream()) -> iolist(). -collect(Z) -> - collect(Z, []). - --spec collect(zstream(), iolist()) -> iolist(). -collect(Z, Acc) -> - receive - {Z, {data, Bin}} -> - collect(Z, [Bin|Acc]) - after 0 -> - reverse(Acc) +-spec dequeue_all_chunks(Z, Opts) -> Result when + Z :: zstream(), + Opts :: #zlib_opts{}, + Result :: {need_dictionary, integer(), iolist()} | + iolist(). +dequeue_all_chunks(Z, Opts) -> + dequeue_all_chunks_1(Z, Opts, []). +dequeue_all_chunks_1(Z, Opts, Output) -> + case dequeue_next_chunk(Z, Opts) of + {need_dictionary, _, _} = NeedDict -> + NeedDict; + {continue, Chunk} -> + dequeue_all_chunks_1(Z, Opts, append_iolist(Output, Chunk)); + {finished, Chunk} -> + append_iolist(Output, Chunk) end. --spec flush(zstream()) -> 'ok'. -flush(Z) -> - receive - {Z, {data,_}} -> - flush(Z) - after 0 -> - ok +-spec dequeue_next_chunk(Z, Opts) -> Result when + Z :: zstream(), + Opts :: #zlib_opts{}, + Result :: {need_dictionary, integer(), iolist()} | + {continue, iolist()} | + {finished, iolist()}. +dequeue_next_chunk(Z, Opts) -> + Method = Opts#zlib_opts.method, + IChSz = Opts#zlib_opts.input_chunk_size, + OChSz = Opts#zlib_opts.output_chunk_size, + Flush = Opts#zlib_opts.flush, + Method(Z, IChSz, OChSz, Flush). + +-spec append_iolist(IO, D) -> iolist() when + IO :: iodata(), + D :: iodata(). +append_iolist([], D) when is_list(D) -> D; +append_iolist([], D) -> [D]; +append_iolist(IO, []) -> IO; +append_iolist(IO, [D]) -> [IO, D]; +append_iolist(IO, D) -> [IO, D]. + +%% inflate/2 and friends are documented as throwing an error on Z_NEED_DICT +%% rather than simply returning something to that effect, and deflateParams/3 +%% may flush behind the scenes. This requires us to stow away our current +%% progress in the handle and resume from that point on our next call. +%% +%% Generally speaking this is either a refc binary or nothing at all, so it's +%% pretty cheap. + +-spec save_progress(Z, Kind, Output) -> ok when + Z :: zstream(), + Kind :: inflate | deflate, + Output :: iolist(). +save_progress(Z, Kind, Output) -> + ok = setStash_nif(Z, {Kind, Output}). + +-spec restore_progress(Z, Kind) -> iolist() when + Z :: zstream(), + Kind :: inflate | deflate. +restore_progress(Z, Kind) -> + case getStash_nif(Z) of + {ok, {Kind, Output}} -> + ok = clearStash_nif(Z), + Output; + empty -> + [] end. - -arg_flush(none) -> ?Z_NO_FLUSH; + +-spec clearStash_nif(Z) -> ok when + Z :: zstream(). +clearStash_nif(_Z) -> + erlang:nif_error(undef). + +-spec setStash_nif(Z, Term) -> ok when + Z :: zstream(), + Term :: term(). +setStash_nif(_Z, _Term) -> + erlang:nif_error(undef). + +-spec getStash_nif(Z) -> {ok, term()} | empty when + Z :: zstream(). +getStash_nif(_Z) -> + erlang:nif_error(undef). + +%% The 'proplists' module isn't preloaded so we can't rely on its existence. +proplist_get_value([], _Name, DefVal) -> DefVal; +proplist_get_value([{Name, Value} | _Opts], Name, _DefVal) -> Value; +proplist_get_value([_Head | Opts], Name, DefVal) -> + proplist_get_value(Opts, Name, DefVal). + +arg_flush(none) -> ?Z_NO_FLUSH; %% ?Z_PARTIAL_FLUSH is deprecated in zlib -- deliberately not included. -arg_flush(sync) -> ?Z_SYNC_FLUSH; -arg_flush(full) -> ?Z_FULL_FLUSH; -arg_flush(finish) -> ?Z_FINISH; -arg_flush(_) -> erlang:error(badarg). +arg_flush(sync) -> ?Z_SYNC_FLUSH; +arg_flush(full) -> ?Z_FULL_FLUSH; +arg_flush(finish) -> ?Z_FINISH; +arg_flush(_) -> erlang:error(bad_flush_mode). arg_level(none) -> ?Z_NO_COMPRESSION; arg_level(best_speed) -> ?Z_BEST_SPEED; arg_level(best_compression) -> ?Z_BEST_COMPRESSION; arg_level(default) -> ?Z_DEFAULT_COMPRESSION; arg_level(Level) when is_integer(Level), Level >= 0, Level =< 9 -> Level; -arg_level(_) -> erlang:error(badarg). - +arg_level(_) -> erlang:error(bad_compression_level). + arg_strategy(filtered) -> ?Z_FILTERED; arg_strategy(huffman_only) -> ?Z_HUFFMAN_ONLY; arg_strategy(rle) -> ?Z_RLE; arg_strategy(default) -> ?Z_DEFAULT_STRATEGY; -arg_strategy(_) -> erlang:error(badarg). +arg_strategy(_) -> erlang:error(bad_compression_strategy). arg_method(deflated) -> ?Z_DEFLATED; -arg_method(_) -> erlang:error(badarg). +arg_method(_) -> erlang:error(bad_compression_method). + +arg_eos_behavior(error) -> ?EOS_BEHAVIOR_ERROR; +arg_eos_behavior(reset) -> ?EOS_BEHAVIOR_RESET; +arg_eos_behavior(cut) -> ?EOS_BEHAVIOR_CUT; +arg_eos_behavior(_) -> erlang:error(bad_eos_behavior). -spec arg_bitsz(zwindowbits()) -> zwindowbits(). arg_bitsz(Bits) when is_integer(Bits) andalso - ((8 =< Bits andalso Bits < 48) orelse - (-15 =< Bits andalso Bits =< -8)) -> + ((8 =< Bits andalso Bits < 48) orelse + (-15 =< Bits andalso Bits =< -8)) -> Bits; -arg_bitsz(_) -> erlang:error(badarg). +arg_bitsz(_) -> erlang:error(bad_windowbits). -spec arg_mem(zmemlevel()) -> zmemlevel(). arg_mem(Level) when is_integer(Level), 1 =< Level, Level =< 9 -> Level; -arg_mem(_) -> erlang:error(badarg). - -call(Z, Cmd, Arg) -> - try port_control(Z, Cmd, Arg) of - [0|Res] -> list_to_atom(Res); - [1|Res] -> - flush(Z), - erlang:error(list_to_atom(Res)); - [2,A,B,C,D] -> - (A bsl 24)+(B bsl 16)+(C bsl 8)+D; - [3,A,B,C,D] -> - erlang:error({need_dictionary,(A bsl 24)+(B bsl 16)+(C bsl 8)+D}); - [4, _, _, _, _] -> - inflate_has_more - catch - error:badarg -> %% Rethrow loses port_control from stacktrace. - erlang:error(badarg) - end. +arg_mem(_) -> erlang:error(bad_memlevel). -reverse(X) -> - reverse(X, []). +-spec enqueue_input(Z, IOData) -> ok when + Z :: zstream(), + IOData :: iodata(). +enqueue_input(Z, IOData) -> + enqueue_input_1(Z, erlang:iolist_to_iovec(IOData)). + +enqueue_input_1(_Z, []) -> + ok; +enqueue_input_1(Z, IOVec) -> + case enqueue_nif(Z, IOVec) of + {continue, Remainder} -> enqueue_input_1(Z, Remainder); + ok -> ok + end. -reverse([H|T], Y) -> - reverse(T, [H|Y]); -reverse([], X) -> - X. +enqueue_nif(_Z, _IOVec) -> + erlang:nif_error(undef). diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl index 44d7f63387..db993abe52 100644 --- a/erts/test/erlexec_SUITE.erl +++ b/erts/test/erlexec_SUITE.erl @@ -59,7 +59,7 @@ otp_8209(Config) when is_list(Config) -> {ok,[[PName]]} = init:get_argument(progname), SNameS = "erlexec_test_01", SName = list_to_atom(SNameS++"@"++ - hd(tl(string:tokens(atom_to_list(node()),"@")))), + hd(tl(string:lexemes(atom_to_list(node()),"@")))), Cmd = PName ++ " dummy_param -sname "++SNameS++" -setcookie "++ atom_to_list(erlang:get_cookie()), open_port({spawn,Cmd},[]), @@ -75,7 +75,7 @@ cleanup_node(SNameS,0) -> {error, {would_not_die,list_to_atom(SNameS)}}; cleanup_node(SNameS,N) -> SName = list_to_atom(SNameS++"@"++ - hd(tl(string:tokens(atom_to_list(node()),"@")))), + hd(tl(string:lexemes(atom_to_list(node()),"@")))), case rpc:call(SName,init,stop,[]) of {badrpc,_} -> ok; @@ -322,7 +322,7 @@ zdbbl_dist_buf_busy_limit(Config) when is_list(Config) -> {ok,[[PName]]} = init:get_argument(progname), SNameS = "erlexec_test_02", SName = list_to_atom(SNameS++"@"++ - hd(tl(string:tokens(atom_to_list(node()),"@")))), + hd(tl(string:lexemes(atom_to_list(node()),"@")))), Cmd = PName ++ " -sname "++SNameS++" -setcookie "++ atom_to_list(erlang:get_cookie()) ++ " +zdbbl " ++ integer_to_list(LimKB), @@ -400,7 +400,7 @@ emu_args(CmdLineArgs) -> {ok,[[Erl]]} = init:get_argument(progname), EmuCL = os:cmd(Erl ++ " -emu_args_exit " ++ CmdLineArgs), io:format("EmuCL = ~ts", [EmuCL]), - split_emu_clt(string:tokens(EmuCL, [$ ,$\t,$\n,$\r])). + split_emu_clt(string:lexemes(EmuCL, [$ ,$\t,$\n,$\r])). split_emu_clt(EmuCLT) -> split_emu_clt(EmuCLT, [], [], [], emu). diff --git a/erts/test/install_SUITE.erl b/erts/test/install_SUITE.erl index d6c6d6f30e..324b398caa 100644 --- a/erts/test/install_SUITE.erl +++ b/erts/test/install_SUITE.erl @@ -580,7 +580,7 @@ end_per_testcase(_Case, _Config) -> ok. make_dirs(Root, Suffix) -> - do_make_dirs(Root, string:tokens(Suffix, [$/])). + do_make_dirs(Root, string:lexemes(Suffix, [$/])). do_make_dirs(_Root, []) -> ""; @@ -709,4 +709,4 @@ join("") -> join([""|Ds]) -> join(Ds); join([D|Ds]) -> - "/" ++ string:strip(D, both, $/) ++ join(Ds). + "/" ++ string:trim(D, both, [$/]) ++ join(Ds). diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl index 624e5484ba..3081b58835 100644 --- a/erts/test/nt_SUITE.erl +++ b/erts/test/nt_SUITE.erl @@ -117,7 +117,7 @@ wait_for_node(Name) -> do_wait_for_it(FullName,30). make_full_name(Name) -> - [_,Suffix] = string:tokens(atom_to_list(node()),"@"), + [_,Suffix] = string:lexemes(atom_to_list(node()),"@"), list_to_atom(Name ++ "@" ++ Suffix). @@ -171,7 +171,7 @@ service_env(Config) when is_list(Config) -> ["ERLSRV_SERVICE_NAME"]), "erlsrv.exe" = filename:basename( hd( - string:tokens( + string:lexemes( rpc:call(make_full_name(Name), os, getenv, diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl index 54fcfd935f..3abe45c141 100644 --- a/erts/test/otp_SUITE.erl +++ b/erts/test/otp_SUITE.erl @@ -336,7 +336,7 @@ not_recommended_calls(Config, Apps0, MFA) -> _ -> AppStrings = [atom_to_list(A) || A <- SkippedApps], Mess = io_lib:format("Application(s) not present: ~s\n", - [string:join(AppStrings, ", ")]), + [lists:join(", ", AppStrings)]), {comment, Mess} end; _ -> @@ -463,7 +463,7 @@ runtime_dependencies(Config) -> have_rdep(_App, [], _Dep) -> false; have_rdep(App, [RDep | RDeps], Dep) -> - [AppStr, _VsnStr] = string:tokens(RDep, "-"), + [AppStr, _VsnStr] = string:lexemes(RDep, "-"), case Dep == list_to_atom(AppStr) of true -> io:format("~p -> ~s~n", [App, RDep]), diff --git a/erts/test/run_erl_SUITE.erl b/erts/test/run_erl_SUITE.erl index fe1ccba1e2..08edd930b4 100644 --- a/erts/test/run_erl_SUITE.erl +++ b/erts/test/run_erl_SUITE.erl @@ -255,7 +255,7 @@ do_run_erl(Config, Case, Opt) -> net_kernel:monitor_nodes(true), open_port({spawn,Cmd}, []), - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), Node = list_to_atom(NodeName++"@"++Host), receive diff --git a/erts/test/system_smoke.spec b/erts/test/system_smoke.spec index 99092c1dab..bcb727b6d9 100644 --- a/erts/test/system_smoke.spec +++ b/erts/test/system_smoke.spec @@ -1,4 +1,3 @@ -{suites,"../system_test",[ethread_SUITE]}. {cases,"../system_test",otp_SUITE, [undefined_functions, deprecated_not_in_obsolete, diff --git a/erts/test/upgrade_SUITE.erl b/erts/test/upgrade_SUITE.erl index a5639d927d..31ceb06314 100644 --- a/erts/test/upgrade_SUITE.erl +++ b/erts/test/upgrade_SUITE.erl @@ -287,7 +287,7 @@ create_relfile(Node,CreateDir,RelName0,RelVsn) -> true -> case filename:split(Path) -- SplitLibDir of [AppVsn,"ebin"] -> - case string:tokens(AppVsn,"-") of + case string:lexemes(AppVsn,"-") of [AppStr,Vsn] -> App = list_to_atom(AppStr), case lists:member(App,Exclude) of diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index d474c71c4f..cd7a894eb5 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -132,22 +132,9 @@ core_search_conf(RunByTS, DBTop, XDir) -> file_inspect(#core_search_conf{file = File}, Core) -> FRes0 = os:cmd(File ++ " " ++ Core), - FRes = case string:str(FRes0, Core) of - 0 -> - FRes0; - S -> - L = length(FRes0), - E = length(Core), - case S of - 1 -> - lists:sublist(FRes0, E+1, L+1); - _ -> - lists:sublist(FRes0, 1, S-1) - ++ - " " - ++ - lists:sublist(FRes0, E+1, L+1) - end + FRes = case string:split(FRes0, Core) of + [S1] -> S1; + [S1,S2] -> lists:flatten(S1 ++ " " ++ S2) end, case re:run(FRes, "text|ascii", [caseless,{capture,none}]) of match -> @@ -194,9 +181,6 @@ mod_time_list(F) -> [0,0,0,0,0,0] end. -str_strip(S) -> - string:strip(string:strip(string:strip(S), both, $\n), both, $\r). - dump_core(#core_search_conf{ cerl = false }, _) -> ok; dump_core(_, {ignore, _Core}) -> @@ -232,7 +216,7 @@ format_core(#core_search_conf{file = false}, Core, Ignore) -> io:format(" ~s~s " ++ time_fstr() ++ "~s~n", [Ignore, Core] ++ mod_time_list(Core)); format_core(#core_search_conf{file = File}, Core, Ignore) -> - FRes = str_strip(os:cmd(File ++ " " ++ Core)), + FRes = string:trim(os:cmd(File ++ " " ++ Core)), case catch re:run(FRes, Core, [caseless,{capture,none}]) of match -> io:format(" ~s~s " ++ time_fstr() ++ "~n", diff --git a/erts/vsn.mk b/erts/vsn.mk index 05f3b4364e..1c6472a0ab 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 9.0 +VSN = 9.1.1 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |